#!/usr/bin/env python3
"""Fetch from all 5 sources and import into DB."""

import sys
sys.path.insert(0, "src")

from ietf_analyzer.config import Config
from ietf_analyzer.sources import FETCHERS, get_fetcher
from ietf_analyzer.db import Database
from ietf_analyzer.models import Draft
from rich.console import Console

console = Console()
cfg = Config.load()
db = Database(cfg)

# Only fetch from new sources (IETF and W3C already done recently)
sources_to_fetch = ["etsi", "itu", "iso"]

total_new = 0
for source_name in sources_to_fetch:
    console.print(f"\n[bold blue]{'='*60}[/]")
    console.print(f"[bold blue]Fetching from {source_name.upper()}...[/]")
    console.print(f"[bold blue]{'='*60}[/]")

    fetcher = get_fetcher(source_name, cfg)
    try:
        docs = fetcher.search(cfg.search_keywords)
        console.print(f"  Found {len(docs)} documents")

        new_count = 0
        for doc in docs:
            existing = db.get_draft(doc.name)
            if existing:
                continue
            new_count += 1

            # Convert to Draft
            draft = Draft(
                name=doc.name,
                rev="01",
                title=doc.title,
                abstract=doc.abstract,
                source=doc.source,
                source_id=doc.source_id,
                source_url=doc.source_url,
                time=doc.time,
                doc_status=doc.doc_status,
                full_text=doc.full_text,
            )
            db.upsert_draft(draft)

        console.print(f"  [green]Imported {new_count} new documents[/]")
        total_new += new_count
    except Exception as e:
        console.print(f"  [red]Error: {e}[/]")
        import traceback
        traceback.print_exc()
    finally:
        fetcher.close()

console.print(f"\n[bold green]Total new documents: {total_new}[/]")

# Final stats
import sqlite3
conn = sqlite3.connect(cfg.db_path)
rows = conn.execute("SELECT source, COUNT(*) FROM drafts GROUP BY source ORDER BY source").fetchall()
console.print("\n[bold]Database by source:[/]")
for source, count in rows:
    console.print(f"  {source}: {count}")
total = conn.execute("SELECT COUNT(*) FROM drafts").fetchone()[0]
console.print(f"  [bold]Total: {total}[/]")
conn.close()