collect_bitcoin_docs.py

⬇️ Download ← Back
#!/usr/bin/env python3
"""
Bitcoin Operations Dataset Collector
Gathers documentation for fine-tuning a Bitcoin AI assistant.
"""

import os
import json
import requests
from pathlib import Path
from urllib.parse import urljoin

# Bitcoin operations documentation sources
DOCS_SOURCES = {
    "bitcoin_core": "https://developer.bitcoin.org",
    "lightning_cln": "https://lightning.engineering/docs",
    "lightning_spec": "https://github.com/lightning/bolts",
    "mining": "https://github.com/bitcoin/bitcoin/blob/master/doc/mining.md",
    "bitcoinops": "https://bitcoinops.org",
    "lnbook": "https://lnbook.org",
}

def create_dataset_structure(base_dir):
    """Create directory structure for organized dataset."""
    categories = [
        "lightning_network",
        "bitcoin_core",
        "mining",
        "node_operations",
        "wallet_security",
        "bitcoin_scripting",
    ]
    for cat in categories:
        Path(base_dir / cat).mkdir(parents=True, exist_ok=True)

def download_and_parse(url, output_dir, category):
    """Download documentation and parse into training format."""
    try:
        response = requests.get(url, timeout=30)
        if response.status_code == 200:
            # Basic text extraction
            text = response.text.replace('<br>', '\n').replace('</p>', '\n\n')
            # Save as raw doc
            filename = f"{category}_{hash(url) % 10000}.txt"
            with open(output_dir / filename, 'w', encoding='utf-8') as f:
                f.write(text)
            return True
    except Exception as e:
        print(f"Failed to fetch {url}: {e}")
    return False

def create_finetuning_format(docs_dir, output_path):
    """Convert collected docs into fine-tuning format."""
    training_data = []
    
    for doc_file in docs_dir.glob("*.txt"):
        with open(doc_file, 'r', encoding='utf-8') as f:
            content = f.read()
        
        # Create Q&A pairs from documentation
        paragraphs = content.split('\n\n')
        for i, para in enumerate(paragraphs):
            if len(para.strip()) < 50:  # Skip short paragraphs
                continue
                
            # Create instruction-response pairs
            training_data.append({
                "instruction": f"Explain: {para[:100]}...",
                "input": "",
                "output": para
            })
    
    with open(output_path, 'w') as f:
        json.dump(training_data, f, indent=2)
    
    return len(training_data)

if __name__ == "__main__":
    base_dir = Path(__file__).parent
    create_dataset_structure(base_dir)
    print("Dataset structure created. Starting collection...")
    
    # Collect documentation
    for category, url in DOCS_SOURCES.items():
        output_dir = base_dir / category
        print(f"Collecting {category} from {url}")
        download_and_parse(url, output_dir, category)
    
    # Convert to fine-tuning format
    print("Converting to fine-tuning format...")
    total_examples = create_finetuning_format(
        base_dir, 
        base_dir / "bitcoin_finetuning.json"
    )
    print(f"Created {total_examples} training examples")
PYTHON collect_bitcoin_docs.py
