#!/usr/bin/env python3
"""
Comprehensive Bitcoin Operations Dataset Collector
Gathers all Bitcoin-related documentation for fine-tuning the ultimate Bitcoin AI assistant.
Covers:
- BIPs (Bitcoin Improvement Proposals)
- Lightning Network BOLT specifications
- LND (Lightning Network Daemon) documentation
- CLN (Core Lightning) documentation
- Bitcoin Core documentation
- Mining documentation
- Wallet security
- Bitcoin scripting
- Network protocols
"""
import os
import json
import re
import time
import requests
from pathlib import Path
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor, as_completed
from bs4 import BeautifulSoup
# Comprehensive source mapping
SOURCES = {
# Bitcoin Improvement Proposals (mediawiki format)
"bips": {
"base": "https://raw.githubusercontent.com/bitcoin/bips/master/bip-",
"files": [f"{i:04d}.mediawiki" for i in range(1, 351)], # BIP-0001 to BIP-0350
"category": "bips"
},
# Lightning Network Protocol Specifications
"bolts": {
"base": "https://raw.githubusercontent.com/lightning/bolts/master/",
"files": ["00-peer-protocol.md", "01-messaging.md", "02-peer-channel-logic.md",
"03-transactions.md", "04-on-chain-elements.md", "05-encryption.md",
"06-resource-requirements.md", "07-tlv.md", "08-transport.md"],
"category": "lightning_protocol"
},
# LND Documentation
"lnd_docs": {
"base": "https://raw.githubusercontent.com/lightningnetwork/lnd/master/docs/",
"category": "lnd"
},
# Core Lightning Documentation
"cln_docs": {
"base": "https://raw.githubusercontent.com/ElementsProject/lightning/master/",
"category": "cln"
},
# Bitcoin Core Documentation
"bitcoin_core": {
"base": "https://raw.githubusercontent.com/bitcoin/bitcoin/master/doc/",
"category": "bitcoin_core"
},
# Mining Documentation
"mining": {
"base": "https://raw.githubusercontent.com/bitcoin/bitcoin/master/doc/",
"category": "mining"
},
# Wallet Security
"wallet_security": {
"base": "https://github.com/bitcoin/bitcoin/blob/master/doc/",
"category": "wallet_security"
},
# Bitcoin Scripting
"scripting": {
"base": "https://github.com/bitcoin/bitcoin/blob/master/doc/",
"category": "scripting"
},
# Real-time Data Sources
"real_time_data": {
"base": "https://mempool.space/api",
"category": "real_time"
},
# Community Knowledge
"community_knowledge": {
"base": "https://bitcoin.stackexchange.com",
"category": "community"
},
# Security Hardening
"security": {
"base": "https://github.com/bitcoin/bitcoin/blob/master/doc/",
"category": "security"
},
# Privacy Operations
"privacy": {
"base": "https://github.com/zkSNACKs/WalletWasabi/blob/master/doc/",
"category": "privacy"
},
# Lightning Plugins
"lightning_plugins": {
"base": "https://github.com/ElementsProject/cln-plugins",
"category": "lightning_plugins"
}
}
class BitcoinDocCollector:
def __init__(self, base_dir="bitcoin_data"):
self.base_dir = Path(base_dir)
self.session = requests.Session()
self.session.headers.update({
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
})
self.create_directories()
def create_directories(self):
"""Create organized directory structure."""
categories = [
"bips",
"lightning_protocol",
"lnd",
"cln",
"bitcoin_core",
"mining",
"wallet_security",
"scripting",
"network_protocols",
"real_time",
"community",
"security",
"privacy",
"lightning_plugins",
"troubleshooting"
]
for cat in categories:
self.base_dir.joinpath(cat).mkdir(parents=True, exist_ok=True)
def download_file(self, url, filename, category):
"""Download a single file and save it."""
try:
response = self.session.get(url, timeout=30)
if response.status_code == 200:
filepath = self.base_dir / category / filename
with open(filepath, 'w', encoding='utf-8') as f:
f.write(response.text)
return True
except Exception as e:
print(f"Failed to download {url}: {e}")
return False
def download_bips(self):
"""Download all Bitcoin Improvement Proposals with rate limiting."""
print("Downloading BIPs...")
bip_urls = [
f"{SOURCES['bips']['base']}{f}"
for f in SOURCES['bips']['files']
]
success_count = 0
for i, url in enumerate(bip_urls):
filename = f"bip_{i+1:04d}.md"
if self.download_file(url, filename, "bips"):
success_count += 1
# Better rate limiting - wait longer between batches
if (i + 1) % 10 == 0:
print(f" Downloaded {i+1} BIPs...")
time.sleep(2) # Wait 2 seconds every 10 BIPs
elif (i + 1) % 5 == 0:
time.sleep(1) # Wait 1 second every 5 BIPs
print(f"BIPs complete: {success_count}/{len(bip_urls)}")
def download_bolts(self):
"""Download Lightning Network protocol specifications."""
print("Downloading BOLTs...")
for filename in SOURCES['bolts']['files']:
url = f"{SOURCES['bolts']['base']}{filename}"
self.download_file(url, filename, "lightning_protocol")
time.sleep(1) # Rate limiting
def download_github_markdown(self, repo_path, category, file_patterns=None):
"""Download markdown files from GitHub repositories."""
print(f"Downloading {category} from {repo_path}...")
# For now, download key documentation files
if category == "lnd":
files = ["README.md", "docs.md"]
elif category == "cln":
files = ["README.md", "DOC.md"]
elif category == "bitcoin_core":
files = ["README.md", "release-notes.md"]
elif category == "mining":
files = ["mining.md"]
elif category == "wallet_security":
files = ["wallet-encryption.md", "wallet-backup.md"]
elif category == "scripting":
files = ["developer-notes.md"]
elif category == "security":
files = ["README.md"]
elif category == "privacy":
files = ["README.md"]
elif category == "lightning_plugins":
files = ["README.md"]
else:
files = ["README.md"]
for filename in files:
# Try raw GitHub URL first
raw_url = repo_path.rstrip('/') + '/' + filename
if self.download_file(raw_url, filename, category):
print(f" Downloaded {filename}")
break # Success, no need to try other URLs
def collect_all(self):
"""Collect all Bitcoin documentation."""
print("Starting comprehensive Bitcoin documentation collection...")
# Download BIPs
self.download_bips()
# Download BOLTs
self.download_bolts()
# Download implementation docs
self.download_github_markdown(
SOURCES['lnd_docs']['base'],
"lnd"
)
self.download_github_markdown(
SOURCES['cln_docs']['base'],
"cln"
)
self.download_github_markdown(
SOURCES['bitcoin_core']['base'],
"bitcoin_core"
)
self.download_github_markdown(
SOURCES['mining']['base'],
"mining"
)
# Download security hardening docs
self.download_github_markdown(
SOURCES['security']['base'],
"security"
)
# Download privacy documentation
self.download_github_markdown(
SOURCES['privacy']['base'],
"privacy"
)
# Download Lightning plugin documentation
self.download_github_markdown(
SOURCES['lightning_plugins']['base'],
"lightning_plugins"
)
# Collect real-time data examples
self.collect_real_time_data()
# Generate troubleshooting scenarios
self.generate_troubleshooting_scenarios()
print("Collection phase 1 complete. Starting phase 2...")
def collect_real_time_data(self):
"""Collect examples of real-time Bitcoin data."""
print("Collecting real-time data examples...")
# Mempool.space API examples
mempool_endpoints = {
"mempool_recent": "/api/mempool/recent",
"mempool_stats": "/api/mempool",
"block_tip": "/api/blocks/tip",
"hashrate_3d": "/api/mining/hashrate/3d",
"fee_statistics": "/api/mining/fee-statistics"
}
headers = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "application/json",
"Accept-Language": "en-US,en;q=0.9"
}
for name, endpoint in mempool_endpoints.items():
url = f"{SOURCES['real_time_data']['base']}{endpoint}"
try:
response = requests.get(url, headers=headers, timeout=30)
if response.status_code == 200:
filepath = self.base_dir / "real_time" / f"{name}.json"
with open(filepath, 'w', encoding='utf-8') as f:
json.dump(response.json(), f, indent=2)
print(f" Collected {name}")
else:
print(f" Failed {name}: HTTP {response.status_code}")
except Exception as e:
print(f" Failed to fetch {url}: {e}")
def generate_troubleshooting_scenarios(self):
"""Generate troubleshooting scenarios based on common issues."""
print("Generating troubleshooting scenarios...")
scenarios = [
{
"scenario": "Node won't sync",
"commands": [
"bitcoin-cli getblockchaininfo",
"bitcoin-cli getnetworkinfo",
"bitcoin-cli verifychain"
],
"solutions": [
"Check disk space",
"Verify internet connection",
"Try reindexing with -reindex option"
]
},
{
"scenario": "Lightning channel stuck",
"commands": [
"lncli listchannels",
"lncli decodepayreq <invoice>",
"lncli closedchannels"
],
"solutions": [
"Force close with lncli closechannel --force",
"Wait for cooperative close",
"Check for on-chain resolution"
]
},
{
"scenario": "Disk space running out",
"commands": [
"bitcoin-cli help prune",
"du -sh ~/.bitcoin/",
"bitcoin-cli getblockchaininfo"
],
"solutions": [
"Enable pruning with prune=550",
"Remove old blocks manually",
"Move blockchain to external drive"
]
},
{
"scenario": "Stratum connection failure",
"commands": [
"telnet pool.example.com 3333",
"ping pool.example.com",
"bitcoin-cli help getnetworkinfo"
],
"solutions": [
"Check pool credentials",
"Verify stratum version compatibility",
"Update mining software"
]
},
{
"scenario": "Lightning invoice expired",
"commands": [
"lncli decodepayreq <invoice>",
"lncli listinvoices",
"lncli addinvoice <amount>"
],
"solutions": [
"Create new invoice",
"Check expiry time",
"Use longer expiry for large amounts"
]
},
{
"scenario": "Wallet recovery",
"commands": [
"electrum restore",
"bitcoin-cli dumpprivkey <address>",
"bitcoin-cli importprivkey <key>"
],
"solutions": [
"Use seed phrase to restore",
"Import private keys",
"Verify wallet balance after restore"
]
},
{
"scenario": "Memory/CPU optimization",
"commands": [
"bitcoin-cli help dbcache",
"top -bn1 | grep bitcoin",
"free -m"
],
"solutions": [
"Increase dbcache parameter",
"Enable pruning",
"Use SSD for blockchain data"
]
},
{
"scenario": "Tor setup for Bitcoin",
"commands": [
"sudo apt install tor",
"bitcoin.conf -proxy=127.0.0.1:9050",
"bitcoin-cli getnetworkinfo"
],
"solutions": [
"Install Tor package",
"Configure proxy in bitcoin.conf",
"Restart Bitcoin Core"
]
},
{
"scenario": "Channel rebalancing",
"commands": [
"lightning-cli listchannels",
"lightning-cli channel",
"lightning-cli help channel"
],
"solutions": [
"Use internal rebalance",
"Open new channel with balance",
"Use c-lightning plugins"
]
},
{
"scenario": "Multi-sig setup",
"commands": [
"bitcoin-cli createwallet multisig",
"bitcoin-cli createmultisig 2 '[\"pubkey1\", \"pubkey2\", \"pubkey3\"]'",
"bitcoin-cli importaddress <address>"
],
"solutions": [
"Generate multiple keys",
"Create multi-sig address",
"Import to all wallets"
]
}
]
# Save troubleshooting scenarios
filepath = self.base_dir / "troubleshooting" / "scenarios.json"
with open(filepath, 'w', encoding='utf-8') as f:
json.dump(scenarios, f, indent=2)
print(f"Generated {len(scenarios)} troubleshooting scenarios")
if __name__ == "__main__":
collector = BitcoinDocCollector()
collector.collect_all()
print("Documentation collection complete!")