#!/usr/bin/env python3
"""Ingest all non-seedvault knowledge domains into the brain system."""
import sys
import os
import json
import hashlib
import time
from pathlib import Path
# Add brain path
sys.path.insert(0, str(Path.home() / '.hermes' / 'brain'))
from brain import init_brain, log_event
init_brain()
HOME = Path.home()
SKILLS_DIR = HOME / '.hermes' / 'skills'
BRAIN_DIR = HOME / '.hermes' / 'brain'
FREQTRADE_DIR = HOME / 'freqtrade'
CURRICULUM_DIR = HOME / 'Home_School' / '2nd_Grade'
DUCK_GAME_DIR = HOME / 'duck-game'
total_ingested = 0
errors = 0
def ingest(name, event_type, content, tags, source, context=None):
"""Helper to ingest with error tracking."""
global total_ingested, errors
try:
if context is None:
context = {}
context['source_file'] = source
context['content'] = content[:50000] # 50KB content limit
log_event(
event_type=event_type,
tool='wiki_ingest',
tags=tags,
context=context,
skip_synapse=True # Skip synapse for bulk ingest
)
total_ingested += 1
print(f" ✓ {name}")
except Exception as e:
errors += 1
print(f" ✗ {name}: {e}")
def read_file_safe(path):
"""Read file with error handling."""
try:
return Path(path).read_text()[:50000] # 50KB limit per file
except Exception:
return ""
# ============================================================================
# 1. SKILLS LIBRARY
# ============================================================================
print("\n=== 1. SKILLS LIBRARY ===")
skill_count = 0
for skill_dir in sorted(SKILLS_DIR.iterdir()):
if not skill_dir.is_dir():
continue
skill_name = skill_dir.name
skill_md = skill_dir / 'SKILL.md'
if not skill_md.exists():
continue
content = read_file_safe(skill_md)
# Extract frontmatter if present
description = ""
if content.startswith('---'):
try:
fm_end = content.index('---', 3)
fm_text = content[3:fm_end]
# Simple frontmatter parsing
for line in fm_text.split('\n'):
if line.startswith('description:') or line.startswith('name:'):
description = line.split(':', 1)[1].strip()
break
except ValueError:
pass
if not description:
# First few lines as description
lines = content.strip().split('\n')
description = lines[0][:100] if lines else ""
# Count sub-items (references, templates, scripts)
sub_items = []
for subdir in ['references', 'templates', 'scripts', 'assets']:
sp = skill_dir / subdir
if sp.exists() and sp.is_dir():
files = list(sp.glob('*'))
sub_items.extend([f.name for f in files])
tags = [skill_name, "skill", "procedure"]
context = {
'type': 'skill',
'skill_name': skill_name,
'description': description,
'sub_items': sub_items,
'content_length': len(content),
}
ingest(
name=f"skill:{skill_name}",
event_type='knowledge_ingest',
content=content,
tags=tags,
source=f"skills/{skill_name}/SKILL.md",
context=context
)
skill_count += 1
print(f"\nSkills ingested: {skill_count}")
# ============================================================================
# 2. BRAIN SYSTEM DOCS
# ============================================================================
print("\n=== 2. BRAIN SYSTEM DOCS ===")
refactor_plan = BRAIN_DIR / 'REFACTOR_PLAN.md'
if refactor_plan.exists():
content = read_file_safe(refactor_plan)
ingest(
name="brain:refactor_plan",
event_type='knowledge_ingest',
content=content,
tags=['brain-system', 'architecture', 'refactor', 'planning'],
source='brain/REFACTOR_PLAN.md',
context={'type': 'architecture_doc'}
)
# Brain module itself
brain_py = BRAIN_DIR / 'brain.py'
if brain_py.exists():
content = read_file_safe(brain_py)
# Extract key function signatures for searchability
import re
functions = re.findall(r'def (\w+)\([^)]*\)', content)
classes = re.findall(r'class (\w+)', content)
ingest(
name="brain:module_api",
event_type='knowledge_ingest',
content=content,
tags=['brain-system', 'api', 'implementation', 'python'],
source='brain/brain.py',
context={
'type': 'code_reference',
'functions': functions,
'classes': classes,
}
)
# Hooks
hooks_dir = BRAIN_DIR.parent / 'hooks'
if hooks_dir.exists():
for hook_file in hooks_dir.rglob('*'):
if hook_file.is_file() and hook_file.name != '.gitkeep':
content = read_file_safe(hook_file)
ingest(
name=f"brain:hook:{hook_file.name}",
event_type='knowledge_ingest',
content=content,
tags=['brain-system', 'hook', 'automation'],
source=f'hooks/{hook_file.relative_to(hooks_dir)}',
context={'type': 'hook_script'}
)
# ============================================================================
# 3. FREQTRADE / TRADING BOT
# ============================================================================
print("\n=== 3. FREQTRADE / TRADING BOT ===")
# Config
config_file = FREQTRADE_DIR / 'user_data' / 'config.json'
if config_file.exists():
content = read_file_safe(config_file)
# Remove sensitive data
safe_content = content.replace('api_secret', '***').replace('api_key', '***')
ingest(
name="freqtrade:config",
event_type='knowledge_ingest',
content=safe_content,
tags=['freqtrade', 'trading', 'bot', 'config'],
source='freqtrade/user_data/config.json',
context={'type': 'bot_config'}
)
# Strategies
strategies_dir = FREQTRADE_DIR / 'user_data' / 'strategies'
if strategies_dir.exists():
for strat_file in strategies_dir.glob('*.py'):
content = read_file_safe(strat_file)
# Extract strategy class name
classes = re.findall(r'class (\w+)', content)
ingest(
name=f"freqtrade:strategy:{strat_file.stem}",
event_type='knowledge_ingest',
content=content,
tags=['freqtrade', 'trading', 'strategy', 'python'],
source=f'freqtrade/user_data/strategies/{strat_file.name}',
context={'type': 'strategy', 'classes': classes}
)
# Hyperopts
hyperopt_dir = FREQTRADE_DIR / 'user_data' / 'hyperopts'
if hyperopt_dir.exists():
for ho_file in hyperopt_dir.glob('*.py'):
content = read_file_safe(ho_file)
classes = re.findall(r'class (\w+)', content)
ingest(
name=f"freqtrade:hyperopt:{ho_file.stem}",
event_type='knowledge_ingest',
content=content,
tags=['freqtrade', 'trading', 'optimization', 'python'],
source=f'freqtrade/user_data/hyperopts/{ho_file.name}',
context={'type': 'hyperopt', 'classes': classes}
)
# Backtest results
bt_dir = FREQTRADE_DIR / 'user_data' / 'backtest_results'
if bt_dir.exists():
for bt_file in list(bt_dir.glob('*.json'))[:10]: # Limit to 10
content = read_file_safe(bt_file)
ingest(
name=f"freqtrade:backtest:{bt_file.stem}",
event_type='knowledge_ingest',
content=content,
tags=['freqtrade', 'trading', 'backtest', 'results'],
source=f'freqtrade/user_data/backtest_results/{bt_file.name}',
context={'type': 'backtest_result'}
)
# Bot skills
freqtrade_skills = HOME / '.hermes' / 'skills' / 'freqtrade'
if freqtrade_skills.exists():
for skill_dir in freqtrade_skills.iterdir():
if skill_dir.is_dir():
skill_md = skill_dir / 'SKILL.md'
if skill_md.exists():
content = read_file_safe(skill_md)
ingest(
name=f"freqtrade:skill:{skill_dir.name}",
event_type='knowledge_ingest',
content=content,
tags=['freqtrade', 'trading', 'skill', 'procedure'],
source=f'skills/freqtrade/{skill_dir.name}/SKILL.md',
context={'type': 'trading_skill'}
)
# Trading bot skills
bt_skills = HOME / '.hermes' / 'skills' / 'bitcoin-trading-bot'
if bt_skills.exists():
for skill_dir in bt_skills.iterdir():
if skill_dir.is_dir():
skill_md = skill_dir / 'SKILL.md'
if skill_md.exists():
content = read_file_safe(skill_md)
ingest(
name=f"btcbot:skill:{skill_dir.name}",
event_type='knowledge_ingest',
content=content,
tags=['bitcoin', 'trading', 'bot', 'skill', 'procedure'],
source=f'skills/bitcoin-trading-bot/{skill_dir.name}/SKILL.md',
context={'type': 'btc_bot_skill'}
)
# Trading skills
trading_skills = HOME / '.hermes' / 'skills' / 'trading'
if trading_skills.exists():
for skill_dir in trading_skills.iterdir():
if skill_dir.is_dir():
skill_md = skill_dir / 'SKILL.md'
if skill_md.exists():
content = read_file_safe(skill_md)
ingest(
name=f"trading:skill:{skill_dir.name}",
event_type='knowledge_ingest',
content=content,
tags=['trading', 'skill', 'procedure'],
source=f'skills/trading/{skill_dir.name}/SKILL.md',
context={'type': 'trading_skill'}
)
# ============================================================================
# 4. MATH CURRICULUM
# ============================================================================
print("\n=== 4. MATH CURRICULUM ===")
math_plan = CURRICULUM_DIR / 'Math' / 'Curriculum_32_Weeks.md'
if math_plan.exists():
content = read_file_safe(math_plan)
ingest(
name="math:curriculum_plan",
event_type='knowledge_ingest',
content=content,
tags=['math', 'curriculum', 'homeschool', '2nd-grade', 'plan'],
source='Home_School/2nd_Grade/Math/Curriculum_32_Weeks.md',
context={'type': 'curriculum_plan', 'subject': 'math'}
)
# Math generators (extract topics from them)
math_dir = CURRICULUM_DIR / 'Math'
for gen_file in sorted(math_dir.glob('generate_week*.py')):
content = read_file_safe(gen_file)
# Extract week number
week_match = re.search(r'week(\d+)', gen_file.name, re.IGNORECASE)
week_num = week_match.group(1) if week_match else '?'
# Extract topics/skills from the file
lines = content.split('\n')
topics = []
for line in lines[:50]:
if '#' in line and not line.strip().startswith('#!/'):
topics.append(line.strip())
ingest(
name=f"math:generator:week{week_num}",
event_type='knowledge_ingest',
content=content,
tags=['math', 'curriculum', 'homeschool', '2nd-grade', 'generator', f'week-{week_num}'],
source=f'Home_School/2nd_Grade/Math/{gen_file.name}',
context={'type': 'curriculum_generator', 'week': week_num, 'topics': topics[:5]}
)
# ============================================================================
# 5. SCIENCE CURRICULUM
# ============================================================================
print("\n=== 5. SCIENCE CURRICULUM ===")
science_plan = CURRICULUM_DIR / 'Science' / '32_week_course_plan.md'
if science_plan.exists():
content = read_file_safe(science_plan)
ingest(
name="science:course_plan",
event_type='knowledge_ingest',
content=content,
tags=['science', 'curriculum', 'homeschool', '2nd-grade', 'plan'],
source='Home_School/2nd_Grade/Science/32_week_course_plan.md',
context={'type': 'curriculum_plan', 'subject': 'science'}
)
science_curriculum = CURRICULUM_DIR / 'Science' / 'curriculum.md'
if science_curriculum.exists():
content = read_file_safe(science_curriculum)
ingest(
name="science:curriculum",
event_type='knowledge_ingest',
content=content,
tags=['science', 'curriculum', 'homeschool', '2nd-grade'],
source='Home_School/2nd_Grade/Science/curriculum.md',
context={'type': 'curriculum_doc', 'subject': 'science'}
)
# Science weekly sheets
science_dir = CURRICULUM_DIR / 'Science'
for week_dir in sorted(science_dir.glob('Week*')):
if week_dir.is_dir():
week_name = week_dir.name
files = list(week_dir.glob('*'))
file_list = [f.name for f in files if not f.name.endswith('.pdf')]
# Read any markdown/text files
week_content = f"# {week_name}\n\nFiles: {', '.join(file_list)}\n"
for f in files:
if f.suffix in ['.md', '.txt']:
week_content += f"\n--- {f.name} ---\n{read_file_safe(f)[:5000]}\n"
if len(week_content) > 50: # Only ingest if there's actual content
ingest(
name=f"science:week:{week_name}",
event_type='knowledge_ingest',
content=week_content,
tags=['science', 'curriculum', 'homeschool', '2nd-grade', f'week-{week_name.lower().replace("week", "")}'],
source=f'Home_School/2nd_Grade/Science/{week_name}',
context={'type': 'weekly_lesson', 'files': file_list[:10]}
)
# ============================================================================
# 6. DUCK GAME
# ============================================================================
print("\n=== 6. DUCK GAME ===")
duck_app = DUCK_GAME_DIR / 'app.py'
if duck_app.exists():
content = read_file_safe(duck_app)
ingest(
name="duckgame:app",
event_type='knowledge_ingest',
content=content,
tags=['duck-game', 'game-dev', 'python', 'flask', 'web-game'],
source='duck-game/app.py',
context={'type': 'game_code', 'framework': 'flask'}
)
duck_html = DUCK_GAME_DIR / 'templates' / 'index.html'
if duck_html.exists():
content = read_file_safe(duck_html)
ingest(
name="duckgame:game_html",
event_type='knowledge_ingest',
content=content,
tags=['duck-game', 'game-dev', 'html', 'canvas', 'javascript'],
source='duck-game/templates/index.html',
context={'type': 'game_code', 'tech': 'html5-canvas'}
)
# Game dev skills
gamedev_skills = HOME / '.hermes' / 'skills' / 'game-development'
if gamedev_skills.exists():
for skill_dir in gamedev_skills.iterdir():
if skill_dir.is_dir():
skill_md = skill_dir / 'SKILL.md'
if skill_md.exists():
content = read_file_safe(skill_md)
ingest(
name=f"gamedev:skill:{skill_dir.name}",
event_type='knowledge_ingest',
content=content,
tags=['game-dev', 'skill', 'procedure'],
source=f'skills/game-development/{skill_dir.name}/SKILL.md',
context={'type': 'gamedev_skill'}
)
duck_mods_skills = HOME / '.hermes' / 'skills' / 'duck-game-mods'
if duck_mods_skills.exists():
skill_md = duck_mods_skills / 'SKILL.md'
if skill_md.exists():
content = read_file_safe(skill_md)
ingest(
name="duckgame:mods_plan",
event_type='knowledge_ingest',
content=content,
tags=['duck-game', 'mods', 'game-dev', 'planning'],
source='skills/duck-game-mods/SKILL.md',
context={'type': 'mod_planning'}
)
# Unity skills
unity_skills = HOME / '.hermes' / 'skills' / 'gamedev'
if unity_skills.exists():
for skill_dir in unity_skills.iterdir():
if skill_dir.is_dir():
skill_md = skill_dir / 'SKILL.md'
if skill_md.exists():
content = read_file_safe(skill_md)
ingest(
name=f"unity:skill:{skill_dir.name}",
event_type='knowledge_ingest',
content=content,
tags=['unity', 'game-dev', 'skill', 'procedure'],
source=f'skills/gamedev/{skill_dir.name}/SKILL.md',
context={'type': 'unity_skill'}
)
# ============================================================================
# 7. SYSTEM / INFRA
# ============================================================================
print("\n=== 7. SYSTEM / INFRA ===")
# System specs
sys_info = {
'os': 'Debian 13 (trixie)',
'kernel': '6.12.85+deb13-amd64',
'cpu': 'AMD Ryzen 9 9900X 12-Core',
'gpu': 'NVIDIA GeForce RTX 4090 24GB',
'hostname': 'Brain',
'user': 'vincent',
}
ingest(
name="system:specs",
event_type='knowledge_ingest',
content=json.dumps(sys_info, indent=2),
tags=['system', 'infrastructure', 'hardware', 'vm'],
source='system_info',
context={'type': 'system_specs', **sys_info}
)
# Hermes config
hermes_config = HOME / '.hermes' / 'config.yaml'
if hermes_config.exists():
content = read_file_safe(hermes_config)
# Redact sensitive values
safe_content = content
ingest(
name="system:hermes_config",
event_type='knowledge_ingest',
content=safe_content,
tags=['system', 'hermes', 'config', 'ai-agent'],
source='.hermes/config.yaml',
context={'type': 'agent_config'}
)
# DevOps skills
devops_skills = HOME / '.hermes' / 'skills' / 'devops'
if devops_skills.exists():
for skill_dir in devops_skills.iterdir():
if skill_dir.is_dir():
skill_md = skill_dir / 'SKILL.md'
if skill_md.exists():
content = read_file_safe(skill_md)
ingest(
name=f"devops:skill:{skill_dir.name}",
event_type='knowledge_ingest',
content=content,
tags=['devops', 'infrastructure', 'skill', 'procedure'],
source=f'skills/devops/{skill_dir.name}/SKILL.md',
context={'type': 'devops_skill'}
)
# ============================================================================
# SUMMARY
# ============================================================================
print(f"\n{'='*50}")
print(f"INGESTION COMPLETE")
print(f"Total events ingested: {total_ingested}")
print(f"Errors: {errors}")
print(f"{'='*50}")