#!/usr/bin/env python3
"""Full audit of all 20 ELA weeks.
Checks: generation, page counts, word coverage, teacher guide consistency,
swap words in word lists, sort coverage, pattern hunt decoys,
capitalization needs, paragraph label counts, data format mismatches,
detective types, Friday opposite words.
"""
import ast
import os
import re
import sys
import subprocess
import traceback
from typing import List, Dict, Tuple, Any, Optional

BASE = os.path.expanduser("~/Home_School/2nd_Grade/English_Language_Arts")

# Target page counts
PAGE_TARGETS = {"Monday": 4, "Tuesday": 3, "Wednesday": 4, "Thursday": 4, "Friday": 3, "Teacher_Guide": 6}

def extract_words(generator_path: str) -> List[Tuple]:
    """Extract WORDS list from generator."""
    with open(generator_path) as f:
        source = f.read()
    
    # Try AST extraction first
    try:
        tree = ast.parse(source)
        for node in ast.walk(tree):
            if isinstance(node, ast.Assign):
                for target in node.targets:
                    if isinstance(target, ast.Name) and target.id == "WORDS":
                        if isinstance(node.value, (ast.List, ast.Tuple)):
                            words = []
                            for elt in node.value.elts:
                                if isinstance(elt, ast.Tuple):
                                    vals = []
                                    for val in elt.elts:
                                        if isinstance(val, ast.Constant):
                                            vals.append(val.value)
                                    if len(vals) >= 2:
                                        words.append(tuple(vals))
                            return words
    except:
        pass
    
    # Fallback to regex
    words = []
    m = re.search(r'WORDS\s*=\s*\[(.*?)\]', source, re.DOTALL)
    if m:
        for tm in re.finditer(r'\(\s*["\']([^"\']*)["\']\s*,\s*["\']([^"\']*)["\'](?:\s*,\s*["\']([^"\']*)["\'](?:\s*,\s*["\']([^"\']*)["\'])?)?\)', m.group(1)):
            vals = [tm.group(i) for i in range(1, 5) if tm.group(i) is not None]
            if vals:
                words.append(tuple(vals))
    return words

def extract_word_names(words: List[Tuple]) -> List[str]:
    return [w[0] for w in words]

def extract_story(source: str, day_key: str) -> str:
    """Extract story text for a given day."""
    keys = {
        "Monday": [r'MONDAY_STORY\s*=\s*\((.*?)\)', r'MONDAY_STORY\s*=\s*"([^"]*)"'],
        "Thursday": [r'THURSDAY_STORY\s*=\s*\((.*?)\)', r'THURSDAY_STORY\s*=\s*"([^"]*)"'],
        "Friday": [r'FRIDAY_REVIEW_STORY\s*=\s*\((.*?)\)', r'FRIDAY_REVIEW_STORY\s*=\s*"([^"]*)"'],
        "Tuesday": [r'TUESDAY_READING\s*=\s*\((.*?)\)', r'TUESDAY_READING\s*=\s*"([^"]*)"'],
        "Wednesday": [r'WEDNESDAY_READING\s*=\s*\((.*?)\)', r'WEDNESDAY_READING\s*=\s*"([^"]*)"'],
    }
    for pat in keys.get(day_key, []):
        m = re.search(pat, source, re.DOTALL)
        if m:
            text = m.group(1).replace('\\n', ' ').replace('"', '').strip()
            return ' '.join(text.split())
    return ""

def check_words_in_story(story: str, word_names: List[str]) -> Tuple[List[str], List[str]]:
    """Check which words appear in a story."""
    if not story:
        return word_names, []  # all missing if no story
    found = []
    missing = []
    story_lower = story.lower()
    for w in word_names:
        if w.lower() in story_lower:
            found.append(w)
        else:
            missing.append(w)
    return found, missing

def extract_fill_data(source: str) -> List[Tuple]:
    """Extract fill-in-the-blank data."""
    fills = []
    for prefix in ["MONDAY_FILL_DATA", "MONDAY_FILL", "TUESDAY_FILL_DATA", "WEDNESDAY_FILL", "THURSDAY_FILL", "FRIDAY_FILL"]:
        m = re.search(rf'{prefix}\s*=\s*\[(.*?)\]', source, re.DOTALL)
        if m:
            for tm in re.finditer(r'\(\s*["\']([^"\']*)["\']\s*,\s*["\']([^"\']*)["\']\s*\)', m.group(1)):
                fills.append((tm.group(1), tm.group(2)))
    return fills

def extract_teacher_fill_answers(source: str) -> List:
    """Extract teacher fill answers."""
    answers = []
    for prefix in ["TEACHER_MONDAY_FILL_ANSWERS", "TEACHER_WEDNESDAY_FILL_ANSWERS", "TEACHER_THURSDAY_FILL_ANSWERS", "TEACHER_FRIDAY_FILL_ANSWERS"]:
        m = re.search(rf'{prefix}\s*=\s*(\[.*?\])', source)
        if m:
            try:
                answers = ast.literal_eval(m.group(1))
            except:
                answers = []
    return answers

def extract_sort_data(source: str) -> Tuple[List, Dict]:
    """Extract sort data and teacher sort answers."""
    sort_data = []
    sort_answers = {}
    
    for prefix in ["MONDAY_SORT_LABELS", "TEACHER_MONDAY_SORT_ANSWERS", 
                   "TEACHER_TUESDAY_SORT_ANSWERS", "TEACHER_THURSDAY_SORT_ANSWERS",
                   "TEACHER_WEDNESDAY_SORT_ANSWERS"]:
        m = re.search(rf'{prefix}\s*=\s*(\[.*?\])', source, re.DOTALL)
        if m and "LABELS" in prefix:
            try:
                sort_data = ast.literal_eval(m.group(1))
            except:
                pass
        elif m and "ANSWERS" in prefix:
            try:
                sort_answers.update(ast.literal_eval(m.group(1)))
            except:
                pass
    
    return sort_data, sort_answers

def extract_swap_data(source: str) -> List:
    """Extract swap words from Tuesday and Thursday."""
    swap_words = []
    for prefix in ["TUESDAY_SWAP_DATA", "THURSDAY_SWAP_DATA"]:
        m = re.search(rf'{prefix}\s*=\s*\[(.*?)\]', source, re.DOTALL)
        if m:
            for tm in re.finditer(r'\(\s*["\']([^"\']*)["\']\s*,\s*["\']([^"\']*)["\']\s*\)', m.group(1)):
                swap_words.extend([tm.group(1), tm.group(2)])
    return swap_words

def extract_pattern_hunt(source: str) -> Tuple[List, Dict]:
    """Extract pattern hunt words and answers."""
    hunt_words = []
    hunt_answers = {}
    
    for prefix in ["TUESDAY_PATTERN_HUNT_WORDS", "THURSDAY_PATTERN_HUNT_WORDS"]:
        m = re.search(rf'{prefix}\s*=\s*\[(.*?)\]', source, re.DOTALL)
        if m:
            for tm in re.finditer(r'["\']([^"\']*)["\']', m.group(1)):
                hunt_words.append(tm.group(1))
    
    for prefix in ["TEACHER_TUESDAY_PATTERN_HUNT", "TEACHER_THURSDAY_PATTERN_HUNT"]:
        m = re.search(rf'{prefix}\s*=\s*(\{{.*?\}})', source, re.DOTALL)
        if m:
            try:
                hunt_answers = ast.literal_eval(m.group(1))
            except:
                pass
    
    return hunt_words, hunt_answers

def extract_correct_sentences(source: str) -> List[str]:
    """Extract Wednesday correct sentences."""
    sentences = []
    m = re.search(r'WEDNESDAY_CORRECT\s*=\s*\[(.*?)\]', source, re.DOTALL)
    if m:
        for tm in re.finditer(r'["\']([^"\']*)["\']', m.group(1)):
            sentences.append(tm.group(1))
    return sentences

def extract_paragraph_label_data(source: str) -> Tuple[List, List]:
    """Extract paragraph label data and answers."""
    data = []
    answers = []
    
    m = re.search(r'WEDNESDAY_PARAGRAPH_LABEL_DATA\s*=\s*\[(.*?)\]', source, re.DOTALL)
    if m:
        for tm in re.finditer(r'["\']([^"\']*)["\']', m.group(1)):
            data.append(tm.group(1))
    
    m = re.search(r'TEACHER_WEDNESDAY_PARAGRAPH_LABEL_ANSWERS\s*=\s*\[(.*?)\]', source, re.DOTALL)
    if m:
        for tm in re.finditer(r'\(\s*["\']([^"\']*)["\']\s*,\s*["\']([^"\']*)["\']\s*\)', m.group(1)):
            answers.append((tm.group(1), tm.group(2)))
    
    return data, answers

def extract_detective_data(source: str) -> Tuple[List, List]:
    """Extract Wednesday detective data."""
    detective = []
    answers = []
    
    m = re.search(r'WEDNESDAY_NOUN_DETECTIVE\s*=\s*\[(.*?)\]', source, re.DOTALL)
    if m:
        for tm in re.finditer(r'["\']([^"\']*)["\']', m.group(1)):
            detective.append(tm.group(1))
    
    m = re.search(r'TEACHER_WEDNESDAY_DETECTIVE_ANSWERS\s*=\s*\[(.*?)\]', source, re.DOTALL)
    if m:
        for tm in re.finditer(r'\(\s*["\']([^"\']*)["\']\s*,\s*["\']([^"\']*)["\']\s*\)', m.group(1)):
            answers.append((tm.group(1), tm.group(2)))
    
    return detective, answers

def extract_opposite_words(source: str) -> Tuple[List, List]:
    """Extract Friday opposite words."""
    words = []
    answers = []
    
    m = re.search(r'FRIDAY_OPPOSITE_WORDS\s*=\s*\[(.*?)\]', source, re.DOTALL)
    if m:
        for tm in re.finditer(r'["\']([^"\']*)["\']', m.group(1)):
            words.append(tm.group(1))
    
    m = re.search(r'TEACHER_FRIDAY_OPPOSITE_ANSWERS\s*=\s*\[(.*?)\]', source, re.DOTALL)
    if m:
        for tm in re.finditer(r'\(\s*["\']([^"\']*)["\']\s*,\s*["\']([^"\']*)["\']\s*\)', m.group(1)):
            answers.append((tm.group(1), tm.group(2)))
    
    return words, answers

def count_pdf_pages(pdf_path: str) -> int:
    """Count pages in a PDF using pdfinfo or pdftoppm."""
    try:
        result = subprocess.run(["pdfinfo", pdf_path], capture_output=True, text=True, timeout=5)
        for line in result.stdout.split('\n'):
            if line.startswith("Pages:"):
                return int(line.split(":")[1].strip())
    except:
        pass
    try:
        result = subprocess.run(["python3", "-c", 
            f"from PyPDF2 import PdfReader; r=PdfReader('{pdf_path}'); print(len(r.pages))"],
            capture_output=True, text=True, timeout=5)
        if result.stdout.strip():
            return int(result.stdout.strip())
    except:
        pass
    # WeasyPrint fallback: generate HTML and count page-breaks
    return -1

def audit_week(week_num: int) -> Dict:
    """Audit a single week. Returns dict of results."""
    issues = []
    week_dir = os.path.join(BASE, f"Week_{week_num:02d}")
    
    # Find generator
    generator = None
    for f in sorted(os.listdir(week_dir)):
        if f.startswith("generate_week") and f.endswith(".py"):
            if any(x in f for x in ["_backup", "_CORRUPTED", "_new", "_old"]):
                continue
            generator = os.path.join(week_dir, f)
            break
    
    if not generator:
        return {"week": week_num, "status": "FAIL", "issues": ["No generator found"], "page_counts": {}}
    
    result = {"week": week_num, "status": "PASS", "issues": [], "page_counts": {}}
    
    # 1. Try running generator
    try:
        env = os.environ.copy()
        env["PYTHONPATH"] = BASE
        proc = subprocess.run(
            ["python3", os.path.basename(generator)],
            cwd=week_dir, capture_output=True, text=True, timeout=120, env=env
        )
        if proc.returncode != 0:
            stderr_lines = proc.stderr.strip().split('\n')[:10]
            issues.append(f"Generator failed: {'; '.join(stderr_lines)}")
        else:
            stdout_lines = proc.stdout.strip().split('\n')
            if any("error" in l.lower() or "fail" in l.lower() or "traceback" in l.lower() for l in stdout_lines):
                issues.append(f"Generator output errors: {proc.stdout[:200]}")
    except subprocess.TimeoutExpired:
        issues.append("Generator timed out (>120s)")
    except Exception as e:
        issues.append(f"Generator exception: {e}")
    
    # 2. Page counts
    source = open(generator).read()
    
    # Find generated PDFs
    day_dirs = {}
    for day in ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Teacher_Guide"]:
        day_subdir = os.path.join(week_dir, day)
        if os.path.isdir(day_subdir):
            pdfs = [f for f in os.listdir(day_subdir) if f.endswith('.pdf')]
            day_dirs[day] = pdfs
    
    for day, pdfs in day_dirs.items():
        actual = len(pdfs)
        # Count total pages across PDFs
        total_pages = 0
        for pdf in pdfs:
            pdf_path = os.path.join(week_dir, day, pdf)
            pages = count_pdf_pages(pdf_path)
            if pages > 0:
                total_pages += pages
        target = PAGE_TARGETS.get(day, 0)
        result["page_counts"][day] = {"actual_pdfs": actual, "actual_pages": total_pages, "target": target}
        if actual != 1:
            issues.append(f"{day}: Expected 1 PDF, found {actual}")
    
    # 3. Extract words
    words = extract_words(generator)
    word_names = extract_word_names(words)
    
    if len(words) != 12:
        issues.append(f"Expected 12 words, found {len(words)}")
    
    # 4. Check word coverage in stories
    for day_key in ["Monday", "Thursday", "Friday", "Tuesday", "Wednesday"]:
        story = extract_story(source, day_key)
        if story:
            found, missing = check_words_in_story(story, word_names)
            if missing:
                issues.append(f"{day_key} story missing words: {missing}")
    
    # 5. Swap words in word lists
    swap_words = extract_swap_data(source)
    word_set = {w.lower() for w in word_names}
    for sw in swap_words:
        if sw.lower() not in word_set:
            issues.append(f"Swap word '{sw}' not in word list")
    
    # 6. Sort categories cover all 12 words
    _, sort_answers = extract_sort_data(source)
    if sort_answers:
        all_sorted = []
        for cat, cat_words in sort_answers.items():
            if isinstance(cat_words, list):
                all_sorted.extend(cat_words)
        if len(all_sorted) != 12:
            issues.append(f"Sort answers cover {len(all_sorted)} words, expected 12")
        # Check each word appears exactly once
        from collections import Counter
        freq = Counter(w.lower() for w in all_sorted)
        for w, c in freq.items():
            if c != 1:
                issues.append(f"Sort word '{w}' appears {c} times (expected 1)")
        # Check all 12 words covered
        for w in word_names:
            if w.lower() not in freq:
                issues.append(f"Word '{w}' not covered in sort answers")
    
    # 7. Pattern hunt has decoys
    hunt_words, hunt_answers = extract_pattern_hunt(source)
    if hunt_answers and "match" in hunt_answers:
        match_count = len(hunt_answers["match"])
        total = len(hunt_words) if hunt_words else match_count
        if total > 0 and match_count >= total:
            issues.append(f"Pattern hunt: all {match_count}/{total} words are targets (no decoys)")
    
    # 8. Correct sentences need capitalization
    correct_sents = extract_correct_sentences(source)
    for sent in correct_sents:
        if sent and sent[0].isupper():
            issues.append(f"Correct sentence already capitalized: '{sent[:50]}'")
    
    # 9. Paragraph label counts
    pl_data, pl_answers = extract_paragraph_label_data(source)
    if pl_data and pl_answers:
        if len(pl_data) != len(pl_answers):
            issues.append(f"Paragraph labels: {len(pl_data)} data items vs {len(pl_answers)} answers")
    
    # 10. Data format mismatches (tuples rendering as repr)
    if "TEACHER_TUESDAY_SCRAMBLE_ANSWERS" in source:
        m = re.search(r'TEACHER_TUESDAY_SCRAMBLE_ANSWERS\s*=\s*\[(.*?)\]', source, re.DOTALL)
        if m:
            content = m.group(1)
            # Check for tuples that might render as repr
            if re.search(r'\(\s*["\']', content):
                pass  # OK format
    
    # 11. Wednesday detective types valid
    detective, detective_answers = extract_detective_data(source)
    if detective_answers:
        valid_types = {"Compound", "Common", "Proper", "Collective", "Abstract", "Concrete",
                       "Singular", "Plural", "count", "mass", "noun", "verb", "adjective",
                       "adverb", "subject", "object", "predicate nominative",
                       "subject complement"}
        for _, ans_type in detective_answers:
            if ans_type.strip().lower() not in {t.lower() for t in valid_types}:
                issues.append(f"Detective answer type '{ans_type}' may be invalid")
    
    # 12. Friday opposite words in word list
    opp_words, opp_answers = extract_opposite_words(source)
    if opp_words:
        for w in opp_words:
            if w.lower() not in word_set:
                issues.append(f"Friday opposite word '{w}' not in word list")
    
    # Check teacher guide fill answers consistency
    fills = extract_fill_data(source)
    if fills:
        for sentence, answer in fills:
            if answer.lower() not in word_set:
                issues.append(f"Fill answer '{answer}' not in word list")
    
    result["issues"] = issues
    result["status"] = "FAIL" if issues else "PASS"
    return result

def main():
    print("=" * 80)
    print("ELA WEEK AUDIT - ALL 20 WEEKS")
    print("=" * 80)
    
    all_results = []
    total_issues = 0
    
    for week_num in range(1, 21):
        result = audit_week(week_num)
        all_results.append(result)
        status = result["status"]
        issues = result["issues"]
        total_issues += len(issues)
        
        print(f"\n{'='*60}")
        print(f"Week {week_num}: {status}")
        print(f"{'='*60}")
        
        if issues:
            for i, issue in enumerate(issues, 1):
                print(f"  {i}. {issue}")
        
        pc = result.get("page_counts", {})
        if pc:
            print(f"  Page counts:")
            for day, info in pc.items():
                target = info["target"]
                actual = info.get("actual_pages", "?")
                pdfs = info.get("actual_pdfs", "?")
                print(f"    {day}: {pdfs} PDF(s), {actual} pages (target: {target})")
    
    print(f"\n{'='*80}")
    print(f"AUDIT SUMMARY")
    print(f"{'='*80}")
    
    passed = sum(1 for r in all_results if r["status"] == "PASS")
    failed = sum(1 for r in all_results if r["status"] == "FAIL")
    
    print(f"  PASS: {passed}/20")
    print(f"  FAIL: {failed}/20")
    print(f"  Total issues: {total_issues}")
    
    print(f"\nFailed weeks:")
    for r in all_results:
        if r["status"] == "FAIL":
            print(f"  Week {r['week']}: {len(r['issues'])} issues")
    
    print(f"\nIssue breakdown:")
    issue_types = {}
    for r in all_results:
        for issue in r["issues"]:
            # Categorize
            if "Generator" in issue:
                cat = "Generator Error"
            elif "word" in issue.lower() and ("not in" in issue.lower() or "missing" in issue.lower() or "cover" in issue.lower()):
                cat = "Word Coverage / Swap"
            elif "Page" in issue or "PDF" in issue:
                cat = "Page Count"
            elif "capitalized" in issue.lower():
                cat = "Capitalization"
            elif "sort" in issue.lower():
                cat = "Sort Coverage"
            elif "pattern hunt" in issue.lower():
                cat = "Pattern Hunt"
            elif "paragraph" in issue.lower():
                cat = "Paragraph Labels"
            elif "detective" in issue.lower():
                cat = "Detective Type"
            elif "opposite" in issue.lower():
                cat = "Opposite Words"
            elif "Fill" in issue:
                cat = "Fill Answers"
            else:
                cat = "Other"
            issue_types[cat] = issue_types.get(cat, 0) + 1
    
    for cat, count in sorted(issue_types.items(), key=lambda x: -x[1]):
        print(f"  {cat}: {count}")

if __name__ == "__main__":
    main()