PYTHON fetch_woo_products.py

#!/usr/bin/env python3
"""Fetch all WooCommerce product data from the public Store API."""
import json
import re
import html
import requests
from pathlib import Path

API_URL = "https://seedvault.market/wp-json/wc/store/v1/products"
OUTPUT = Path(__file__).parent / "woo_products.json"

def fetch_all():
    all_products = []
    page = 1
    per_page = 50
    
    while True:
        r = requests.get(API_URL, params={"per_page": per_page, "page": page}, timeout=30)
        r.raise_for_status()
        products = r.json()
        
        if not products:
            break
            
        all_products.extend(products)
        print(f"Page {page}: {len(products)} products (total: {len(all_products)})")
        
        # Check if we got fewer than per_page - last page
        if len(products) < per_page:
            break
        page += 1
    
    return all_products


def parse_description(desc_html):
    """Parse product description HTML to extract structured data."""
    text = html.unescape(desc_html)
    
    # Extract germination info
    germination_match = re.search(r'Germination[<>\s]*:[<>\s]*([\d-]+)\s*days?\s*at\s*([\d-]+)°F?', text, re.IGNORECASE)
    
    # Extract harvest time
    harvest_match = re.search(r'Harvest[<>\s]*:[<>\s]*([\d+]+)\s*days?', text, re.IGNORECASE)
    if not harvest_match:
        harvest_match = re.search(r'days?\s*to\s*(?:harvest|maturity|maturity|pod|bloom|fruit)[<>\s]*[:\s]*([\d]+)', text, re.IGNORECASE)
    
    # Extract sowing depth
    depth_match = re.search(r'sow.*?(\d/?\d*)\s*[\'"]?\s*deep', text, re.IGNORECASE)
    if not depth_match:
        depth_match = re.search(r'(\d/?\d*)\s*[\'"]?\s*deep', text, re.IGNORECASE)
    
    # Extract spacing
    spacing_match = re.search(r'(\d+)\s*[\'"]?\s*spacing', text, re.IGNORECASE)
    
    # Extract seed count per packet
    count_match = re.search(r'([\d]+)\s*seeds?\s*per\s*(?:packet|pack)', text, re.IGNORECASE)
    if not count_match:
        count_match = re.search(r'(?:about|approx[.]?\s*)?\s*([\d]+)\s*seeds?', text, re.IGNORECASE)
    
    # Extract soaking/scarification/stratification mentions
    techniques = []
    if re.search(r'soak', text, re.IGNORECASE):
        techniques.append("soaking")
    if re.search(r'scarif', text, re.IGNORECASE):
        techniques.append("scarification")
    if re.search(r'stratif', text, re.IGNORECASE):
        techniques.append("stratification")
    if re.search(r'light.*germinat', text, re.IGNORECASE) or re.search(r'don.t.*cover', text, re.IGNORECASE):
        techniques.append("light_required")
    
    return {
        "germination_days": germination_match.group(1) if germination_match else None,
        "germination_temp": germination_match.group(2) if germination_match else None,
        "harvest_days": harvest_match.group(1) if harvest_match else None,
        "sowing_depth": depth_match.group(1) if depth_match else None,
        "spacing": spacing_match.group(1) if spacing_match else None,
        "seed_count": count_match.group(1) if count_match else None,
        "techniques": techniques,
    }


def main():
    print("Fetching all products from Store API...")
    products = fetch_all()
    OUTPUT.write_text(json.dumps(products, indent=2))
    print(f"\nSaved {len(products)} products to {OUTPUT}")
    
    # Parse descriptions and extract data
    enriched = []
    for p in products:
        data = {
            "id": p.get("id"),
            "name": p.get("name"),
            "slug": p.get("slug"),
            "price": p.get("prices", {}).get("price"),
            "category": [c.get("name") for c in p.get("categories", [])],
            "tags": [t.get("name") for t in p.get("tags", [])],
            "short_description": p.get("short_description", ""),
            "in_stock": p.get("is_in_stock"),
            "stock_text": p.get("stock_availability", {}).get("text"),
        }
        
        desc = p.get("description", "")
        if desc:
            data["parsed"] = parse_description(desc)
        
        enriched.append(data)
    
    # Save enriched data
    enriched_path = Path(__file__).parent / "woo_products_enriched.json"
    enriched_path.write_text(json.dumps(enriched, indent=2))
    print(f"Enriched data saved to {enriched_path}")
    
    # Print summary of extracted data
    print("\n=== DATA EXTRACTION SUMMARY ===")
    with_germination = sum(1 for e in enriched if e.get("parsed", {}).get("germination_days"))
    with_harvest = sum(1 for e in enriched if e.get("parsed", {}).get("harvest_days"))
    with_depth = sum(1 for e in enriched if e.get("parsed", {}).get("sowing_depth"))
    with_techniques = sum(1 for e in enriched if e.get("parsed", {}).get("techniques"))
    
    print(f"Products with germination time: {with_germination}/{len(enriched)}")
    print(f"Products with harvest days: {with_harvest}/{len(enriched)}")
    print(f"Products with sowing depth: {with_depth}/{len(enriched)}")
    print(f"Products with special techniques: {with_techniques}/{len(enriched)}")
    
    # Show techniques found
    all_techniques = {}
    for e in enriched:
        for t in e.get("parsed", {}).get("techniques", []):
            all_techniques[t] = all_techniques.get(t, 0) + 1
    if all_techniques:
        print(f"\nTechniques: {all_techniques}")


if __name__ == "__main__":
    main()