PYTHON combine_datasets.py

#!/usr/bin/env python3
"""
Combine all dataset sources into a single training file.
"""
import json
from pathlib import Path

OUTPUT_FILE = Path(__file__).parent.parent / "data" / "processed" / "dev_finetuning.json"

def main():
    sources = [
        Path(__file__).parent.parent / "data" / "processed" / "dev_finetuning_generated.json",
        Path(__file__).parent.parent / "data" / "processed" / "dev_finetuning_massive.json",
        Path(__file__).parent.parent / "data" / "processed" / "dev_finetuning_real.json",
        Path(__file__).parent.parent / "data" / "processed" / "dev_finetuning_varied.json",
    ]
    
    all_data = []
    for src in sources:
        if src.exists():
            with open(src) as f:
                all_data.extend(json.load(f))
    
    # Remove exact duplicates
    seen = set()
    unique_data = []
    for item in all_data:
        key = json.dumps(item, sort_keys=True)
        if key not in seen:
            seen.add(key)
            unique_data.append(item)
    
    with open(OUTPUT_FILE, "w") as f:
        json.dump(unique_data, f, indent=2)
    
    print(f"Combined {len(all_data)} entries into {len(unique_data)} unique entries")
    print(f"Saved to: {OUTPUT_FILE}")
    print(f"File size: {OUTPUT_FILE.stat().st_size / 1024:.0f} KB")

if __name__ == "__main__":
    main()