PYTHON extract_product_data.py

#!/usr/bin/env python3
"""Extract structured growing data from WooCommerce product descriptions."""
import json
import re
import html
from pathlib import Path

PRODUCTS = Path(__file__).parent / "woo_products.json"
ENTITIES_DIR = Path(__file__).parent / "entities"
OUTPUT = Path(__file__).parent / "extracted_growing_data.json"


def strip_html(text):
    """Remove HTML tags and decode entities."""
    text = re.sub(r'<[^>]+>', ' ', text)
    return html.unescape(text).strip()


def extract_growing_data(desc_html):
    """Extract all growing-related data from product description HTML."""
    text = strip_html(desc_html)
    
    result = {}
    
    # Germination: "Germination: 7–14 days at 75–85°F" or similar
    m = re.search(r'Germination[:.]*\s*([\d–-]+(?:\s*days?|[–-]\s*[\d]+)?)\s*(?:at\s*([\d–-]+)°F?)?', text, re.I)
    if m:
        result['germination_days'] = m.group(1).strip() if m.group(1) else None
        result['germination_temp'] = m.group(2).strip() if m.group(2) else None
    
    # Harvest: "Harvest: 100+ days" or "80 days to maturity"
    m = re.search(r'(?:Harvest|Maturity|Days?\s*to\s*harvest|Days?\s*to\s*maturity|Days?\s*to\s*pod)[:.]*\s*([\d+–]+)\s*days?', text, re.I)
    if m:
        result['harvest_days'] = m.group(1).strip()
    else:
        # "100+ days" anywhere
        m = re.search(r'([\d]+[+–]?)\s*days?\s*(?:to\s*(?:maturity|harvest|pod|bloom|fruit|salad)?|before\s*first)', text, re.I)
        if m:
            result['harvest_days'] = m.group(1).strip()
    
    # Sowing depth: "1/2" deep or "0.5\" deep"
    m = re.search(r'(?:plant|sow)\s+([\d/\.]+)\s*["\']?\s*deep', text, re.I)
    if m:
        result['sowing_depth'] = m.group(1).strip()
    
    # Spacing: "12-18" spacing or "12" apart"
    m = re.search(r'([\d]+(?:\s*[–-]\s*[\d]+)?)\s*["\']?\s*(?:spacing|apart|between\s+plants?)', text, re.I)
    if m:
        result['spacing'] = m.group(1).strip()
    
    # Special techniques
    techniques = []
    if re.search(r'soak.*?(seeds?|overnight|24\s*hours)', text, re.I):
        techniques.append('soaking')
    if re.search(r'scarif', text, re.I):
        techniques.append('scarification')
    if re.search(r'stratif', text, re.I):
        techniques.append('stratification')
    if re.search(r'(?:light\s+(?:to\s+)?germinat|dont\s+cover|surface\s+sow|press\s+into)', text, re.I):
        techniques.append('light_required')
    if re.search(r'indoor.*?start|start.*?indoors', text, re.I):
        techniques.append('start_indoors')
    if re.search(r'succe(?:ssion)?\s*plant', text, re.I):
        techniques.append('succession_planting')
    
    if techniques:
        result['techniques'] = techniques
    
    # Seed count per packet
    m = re.search(r'about\s*([\d,]+)\s*seeds?\s*(?:per\s*packet|per\s*pack|per\s*pkg)', text, re.I)
    if m:
        result['seed_count'] = m.group(1).replace(',', '')
    else:
        m = re.search(r'([\d,]+)\s*seeds?\s*(?:per\s*(?:packet|pack|pkg)|in\s+(?:this|each|per)\s*(?:packet|pack))', text, re.I)
        if m:
            result['seed_count'] = m.group(1).replace(',', '')
    
    # Days to sow: "Sow: X weeks before last frost"
    m = re.search(r'sow[:.]*\s*([\d]+)\s*weeks?\s*(?:before|after)', text, re.I)
    if m:
        result['sow_timing'] = f"{m.group(1)} weeks"
    
    return result


def main():
    with open(PRODUCTS) as f:
        products = json.load(f)
    
    extracted = {}
    
    for p in products:
        slug = p.get('slug')
        desc = p.get('description', '')
        
        if desc:
            data = extract_growing_data(desc)
            if data:
                extracted[slug] = data
                print(f"{slug}: {data}")
    
    OUTPUT.write_text(json.dumps(extracted, indent=2))
    print(f"\nSaved {len(extracted)} entries to {OUTPUT}")
    
    # Compare with wiki entity pages
    wiki_slugs = [f.stem for f in ENTITIES_DIR.glob("*.md") if 'curriculum' not in f.name]
    print(f"\n=== COVERAGE: Wiki entities vs extracted data ===")
    wiki_only = set(wiki_slugs) - set(extracted.keys())
    extracted_only = set(extracted.keys()) - set(wiki_slugs)
    
    if wiki_only:
        print(f"\nWiki entities WITHOUT extracted data: {len(wiki_only)}")
        for s in sorted(wiki_only):
            print(f"  - {s}")
    
    if extracted_only:
        print(f"\nExtracted data WITHOUT wiki entity: {len(extracted_only)}")
        for s in sorted(extracted_only):
            print(f"  - {s}")
    
    # Check which data we got
    print(f"\n=== DATA AVAILABILITY ===")
    for field in ['germination_days', 'harvest_days', 'sowing_depth', 'spacing', 'techniques', 'seed_count']:
        count = sum(1 for v in extracted.values() if field in v)
        print(f"  {field}: {count}/{len(extracted)}")


if __name__ == "__main__":
    main()