#!/usr/bin/env python3 """ Comprehensive Bitcoin Operations Dataset Collector Gathers all Bitcoin-related documentation for fine-tuning the ultimate Bitcoin AI assistant. Covers: - BIPs (Bitcoin Improvement Proposals) - Lightning Network BOLT specifications - LND (Lightning Network Daemon) documentation - CLN (Core Lightning) documentation - Bitcoin Core documentation - Mining documentation - Wallet security - Bitcoin scripting - Network protocols """ import os import json import re import time import requests from pathlib import Path from urllib.parse import urljoin, urlparse from concurrent.futures import ThreadPoolExecutor, as_completed from bs4 import BeautifulSoup # Comprehensive source mapping SOURCES = { # Bitcoin Improvement Proposals (mediawiki format) "bips": { "base": "https://raw.githubusercontent.com/bitcoin/bips/master/bip-", "files": [f"{i:04d}.mediawiki" for i in range(1, 351)], # BIP-0001 to BIP-0350 "category": "bips" }, # Lightning Network Protocol Specifications "bolts": { "base": "https://raw.githubusercontent.com/lightning/bolts/master/", "files": ["00-peer-protocol.md", "01-messaging.md", "02-peer-channel-logic.md", "03-transactions.md", "04-on-chain-elements.md", "05-encryption.md", "06-resource-requirements.md", "07-tlv.md", "08-transport.md"], "category": "lightning_protocol" }, # LND Documentation "lnd_docs": { "base": "https://raw.githubusercontent.com/lightningnetwork/lnd/master/docs/", "category": "lnd" }, # Core Lightning Documentation "cln_docs": { "base": "https://raw.githubusercontent.com/ElementsProject/lightning/master/", "category": "cln" }, # Bitcoin Core Documentation "bitcoin_core": { "base": "https://raw.githubusercontent.com/bitcoin/bitcoin/master/doc/", "category": "bitcoin_core" }, # Mining Documentation "mining": { "base": "https://raw.githubusercontent.com/bitcoin/bitcoin/master/doc/", "category": "mining" }, # Wallet Security "wallet_security": { "base": "https://github.com/bitcoin/bitcoin/blob/master/doc/", "category": "wallet_security" }, # Bitcoin Scripting "scripting": { "base": "https://github.com/bitcoin/bitcoin/blob/master/doc/", "category": "scripting" }, # Real-time Data Sources "real_time_data": { "base": "https://mempool.space/api", "category": "real_time" }, # Community Knowledge "community_knowledge": { "base": "https://bitcoin.stackexchange.com", "category": "community" }, # Security Hardening "security": { "base": "https://github.com/bitcoin/bitcoin/blob/master/doc/", "category": "security" }, # Privacy Operations "privacy": { "base": "https://github.com/zkSNACKs/WalletWasabi/blob/master/doc/", "category": "privacy" }, # Lightning Plugins "lightning_plugins": { "base": "https://github.com/ElementsProject/cln-plugins", "category": "lightning_plugins" } } class BitcoinDocCollector: def __init__(self, base_dir="bitcoin_data"): self.base_dir = Path(base_dir) self.session = requests.Session() self.session.headers.update({ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" }) self.create_directories() def create_directories(self): """Create organized directory structure.""" categories = [ "bips", "lightning_protocol", "lnd", "cln", "bitcoin_core", "mining", "wallet_security", "scripting", "network_protocols", "real_time", "community", "security", "privacy", "lightning_plugins", "troubleshooting" ] for cat in categories: self.base_dir.joinpath(cat).mkdir(parents=True, exist_ok=True) def download_file(self, url, filename, category): """Download a single file and save it.""" try: response = self.session.get(url, timeout=30) if response.status_code == 200: filepath = self.base_dir / category / filename with open(filepath, 'w', encoding='utf-8') as f: f.write(response.text) return True except Exception as e: print(f"Failed to download {url}: {e}") return False def download_bips(self): """Download all Bitcoin Improvement Proposals with rate limiting.""" print("Downloading BIPs...") bip_urls = [ f"{SOURCES['bips']['base']}{f}" for f in SOURCES['bips']['files'] ] success_count = 0 for i, url in enumerate(bip_urls): filename = f"bip_{i+1:04d}.md" if self.download_file(url, filename, "bips"): success_count += 1 # Better rate limiting - wait longer between batches if (i + 1) % 10 == 0: print(f" Downloaded {i+1} BIPs...") time.sleep(2) # Wait 2 seconds every 10 BIPs elif (i + 1) % 5 == 0: time.sleep(1) # Wait 1 second every 5 BIPs print(f"BIPs complete: {success_count}/{len(bip_urls)}") def download_bolts(self): """Download Lightning Network protocol specifications.""" print("Downloading BOLTs...") for filename in SOURCES['bolts']['files']: url = f"{SOURCES['bolts']['base']}{filename}" self.download_file(url, filename, "lightning_protocol") time.sleep(1) # Rate limiting def download_github_markdown(self, repo_path, category, file_patterns=None): """Download markdown files from GitHub repositories.""" print(f"Downloading {category} from {repo_path}...") # For now, download key documentation files if category == "lnd": files = ["README.md", "docs.md"] elif category == "cln": files = ["README.md", "DOC.md"] elif category == "bitcoin_core": files = ["README.md", "release-notes.md"] elif category == "mining": files = ["mining.md"] elif category == "wallet_security": files = ["wallet-encryption.md", "wallet-backup.md"] elif category == "scripting": files = ["developer-notes.md"] elif category == "security": files = ["README.md"] elif category == "privacy": files = ["README.md"] elif category == "lightning_plugins": files = ["README.md"] else: files = ["README.md"] for filename in files: # Try raw GitHub URL first raw_url = repo_path.rstrip('/') + '/' + filename if self.download_file(raw_url, filename, category): print(f" Downloaded {filename}") break # Success, no need to try other URLs def collect_all(self): """Collect all Bitcoin documentation.""" print("Starting comprehensive Bitcoin documentation collection...") # Download BIPs self.download_bips() # Download BOLTs self.download_bolts() # Download implementation docs self.download_github_markdown( SOURCES['lnd_docs']['base'], "lnd" ) self.download_github_markdown( SOURCES['cln_docs']['base'], "cln" ) self.download_github_markdown( SOURCES['bitcoin_core']['base'], "bitcoin_core" ) self.download_github_markdown( SOURCES['mining']['base'], "mining" ) # Download security hardening docs self.download_github_markdown( SOURCES['security']['base'], "security" ) # Download privacy documentation self.download_github_markdown( SOURCES['privacy']['base'], "privacy" ) # Download Lightning plugin documentation self.download_github_markdown( SOURCES['lightning_plugins']['base'], "lightning_plugins" ) # Collect real-time data examples self.collect_real_time_data() # Generate troubleshooting scenarios self.generate_troubleshooting_scenarios() print("Collection phase 1 complete. Starting phase 2...") def collect_real_time_data(self): """Collect examples of real-time Bitcoin data.""" print("Collecting real-time data examples...") # Mempool.space API examples mempool_endpoints = { "mempool_recent": "/api/mempool/recent", "mempool_stats": "/api/mempool", "block_tip": "/api/blocks/tip", "hashrate_3d": "/api/mining/hashrate/3d", "fee_statistics": "/api/mining/fee-statistics" } headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept": "application/json", "Accept-Language": "en-US,en;q=0.9" } for name, endpoint in mempool_endpoints.items(): url = f"{SOURCES['real_time_data']['base']}{endpoint}" try: response = requests.get(url, headers=headers, timeout=30) if response.status_code == 200: filepath = self.base_dir / "real_time" / f"{name}.json" with open(filepath, 'w', encoding='utf-8') as f: json.dump(response.json(), f, indent=2) print(f" Collected {name}") else: print(f" Failed {name}: HTTP {response.status_code}") except Exception as e: print(f" Failed to fetch {url}: {e}") def generate_troubleshooting_scenarios(self): """Generate troubleshooting scenarios based on common issues.""" print("Generating troubleshooting scenarios...") scenarios = [ { "scenario": "Node won't sync", "commands": [ "bitcoin-cli getblockchaininfo", "bitcoin-cli getnetworkinfo", "bitcoin-cli verifychain" ], "solutions": [ "Check disk space", "Verify internet connection", "Try reindexing with -reindex option" ] }, { "scenario": "Lightning channel stuck", "commands": [ "lncli listchannels", "lncli decodepayreq ", "lncli closedchannels" ], "solutions": [ "Force close with lncli closechannel --force", "Wait for cooperative close", "Check for on-chain resolution" ] }, { "scenario": "Disk space running out", "commands": [ "bitcoin-cli help prune", "du -sh ~/.bitcoin/", "bitcoin-cli getblockchaininfo" ], "solutions": [ "Enable pruning with prune=550", "Remove old blocks manually", "Move blockchain to external drive" ] }, { "scenario": "Stratum connection failure", "commands": [ "telnet pool.example.com 3333", "ping pool.example.com", "bitcoin-cli help getnetworkinfo" ], "solutions": [ "Check pool credentials", "Verify stratum version compatibility", "Update mining software" ] }, { "scenario": "Lightning invoice expired", "commands": [ "lncli decodepayreq ", "lncli listinvoices", "lncli addinvoice " ], "solutions": [ "Create new invoice", "Check expiry time", "Use longer expiry for large amounts" ] }, { "scenario": "Wallet recovery", "commands": [ "electrum restore", "bitcoin-cli dumpprivkey
", "bitcoin-cli importprivkey " ], "solutions": [ "Use seed phrase to restore", "Import private keys", "Verify wallet balance after restore" ] }, { "scenario": "Memory/CPU optimization", "commands": [ "bitcoin-cli help dbcache", "top -bn1 | grep bitcoin", "free -m" ], "solutions": [ "Increase dbcache parameter", "Enable pruning", "Use SSD for blockchain data" ] }, { "scenario": "Tor setup for Bitcoin", "commands": [ "sudo apt install tor", "bitcoin.conf -proxy=127.0.0.1:9050", "bitcoin-cli getnetworkinfo" ], "solutions": [ "Install Tor package", "Configure proxy in bitcoin.conf", "Restart Bitcoin Core" ] }, { "scenario": "Channel rebalancing", "commands": [ "lightning-cli listchannels", "lightning-cli channel", "lightning-cli help channel" ], "solutions": [ "Use internal rebalance", "Open new channel with balance", "Use c-lightning plugins" ] }, { "scenario": "Multi-sig setup", "commands": [ "bitcoin-cli createwallet multisig", "bitcoin-cli createmultisig 2 '[\"pubkey1\", \"pubkey2\", \"pubkey3\"]'", "bitcoin-cli importaddress
" ], "solutions": [ "Generate multiple keys", "Create multi-sig address", "Import to all wallets" ] } ] # Save troubleshooting scenarios filepath = self.base_dir / "troubleshooting" / "scenarios.json" with open(filepath, 'w', encoding='utf-8') as f: json.dump(scenarios, f, indent=2) print(f"Generated {len(scenarios)} troubleshooting scenarios") if __name__ == "__main__": collector = BitcoinDocCollector() collector.collect_all() print("Documentation collection complete!")