Christian Nennemann
a46a01bd8c
- Add `ietf auto` command: fetches, analyzes, embeds, extracts ideas, and refreshes gaps across all sources with cost-based auto-approval - Fix SourceDocument→Draft conversion in auto fetch step - Fix gap_analysis method name in auto command - Process all 270 unrated ETSI/ISO/ITU/NIST drafts (761 total, all rated) - Update web UI templates and data layer for multi-source support Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
74 lines
2.2 KiB
Python
74 lines
2.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Fetch from all 5 sources and import into DB."""
|
|
|
|
import sys
|
|
sys.path.insert(0, "src")
|
|
|
|
from ietf_analyzer.config import Config
|
|
from ietf_analyzer.sources import FETCHERS, get_fetcher
|
|
from ietf_analyzer.db import Database
|
|
from ietf_analyzer.models import Draft
|
|
from rich.console import Console
|
|
|
|
console = Console()
|
|
cfg = Config.load()
|
|
db = Database(cfg)
|
|
|
|
# Only fetch from new sources (IETF and W3C already done recently)
|
|
sources_to_fetch = ["etsi", "itu", "iso", "nist"]
|
|
|
|
total_new = 0
|
|
for source_name in sources_to_fetch:
|
|
console.print(f"\n[bold blue]{'='*60}[/]")
|
|
console.print(f"[bold blue]Fetching from {source_name.upper()}...[/]")
|
|
console.print(f"[bold blue]{'='*60}[/]")
|
|
|
|
fetcher = get_fetcher(source_name, cfg)
|
|
try:
|
|
docs = fetcher.search(cfg.search_keywords)
|
|
console.print(f" Found {len(docs)} documents")
|
|
|
|
new_count = 0
|
|
for doc in docs:
|
|
existing = db.get_draft(doc.name)
|
|
if existing:
|
|
continue
|
|
new_count += 1
|
|
|
|
# Convert to Draft
|
|
draft = Draft(
|
|
name=doc.name,
|
|
rev="01",
|
|
title=doc.title,
|
|
abstract=doc.abstract,
|
|
source=doc.source,
|
|
source_id=doc.source_id,
|
|
source_url=doc.source_url,
|
|
time=doc.time,
|
|
doc_status=doc.doc_status,
|
|
full_text=doc.full_text,
|
|
)
|
|
db.upsert_draft(draft)
|
|
|
|
console.print(f" [green]Imported {new_count} new documents[/]")
|
|
total_new += new_count
|
|
except Exception as e:
|
|
console.print(f" [red]Error: {e}[/]")
|
|
import traceback
|
|
traceback.print_exc()
|
|
finally:
|
|
fetcher.close()
|
|
|
|
console.print(f"\n[bold green]Total new documents: {total_new}[/]")
|
|
|
|
# Final stats
|
|
import sqlite3
|
|
conn = sqlite3.connect(cfg.db_path)
|
|
rows = conn.execute("SELECT source, COUNT(*) FROM drafts GROUP BY source ORDER BY source").fetchall()
|
|
console.print("\n[bold]Database by source:[/]")
|
|
for source, count in rows:
|
|
console.print(f" {source}: {count}")
|
|
total = conn.execute("SELECT COUNT(*) FROM drafts").fetchone()[0]
|
|
console.print(f" [bold]Total: {total}[/]")
|
|
conn.close()
|