c/ietf-draft-analyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Platform upgrade: semantic search, citations, readiness, tests, Docker

Browse Source 
Major features added by 5 parallel agent teams:
- Semantic "Ask" (NL queries via FTS5 + embeddings + Claude synthesis)
- Global search across drafts, ideas, authors, gaps
- REST API expansion (14 endpoints, up from 3) with CSV/JSON export
- Citation graph visualization (D3.js, 440 nodes, 2422 edges)
- Standards readiness scoring (0-100 composite from 6 factors)
- Side-by-side draft comparison view with shared/unique analysis
- Annotation system (notes + tags per draft, DB-persisted)
- Docker deployment (Dockerfile + docker-compose with Ollama)
- Scheduled updates (cron script with log rotation)
- Pipeline health dashboard (stage progress bars, cost tracking)
- Test suite foundation (54 pytest tests covering DB, models, web data)

Fixes: compare_drafts() stubbed→working, get_authors_for_draft() bug,
source-aware analysis prompts, config env var overrides + validation,
resilient batch error handling with --retry-failed, observatory --dry-run

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Christian Nennemann
2026-03-07 20:52:56 +01:00
parent da2a989744
commit 757b781c67
33 changed files with 4253 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

401
src/ietf_analyzer/cli.py
View File

@@ -173,6 +173,20 @@ def show(name: str):
else:
console.print("[dim]Not yet rated — run: ietf analyze {name}[/]")
# Readiness score
from .readiness import compute_readiness
readiness = compute_readiness(db, name)
if readiness["score"] > 0:
console.print(f"\n[bold]Standards Readiness: [cyan]{readiness['score']}/100[/][/]")
rtable = Table(show_header=True)
rtable.add_column("Factor", width=20)
rtable.add_column("Value", justify="center", width=10)
rtable.add_column("Points", justify="right", width=8)
rtable.add_column("Detail")
for key, f in readiness["factors"].items():
rtable.add_row(f["label"], f"{f['value']:.2f}", f"+{f['contribution']}", f["detail"])
console.print(rtable)
# Save detailed report too
path = reporter.draft_detail(name)
if path:
@@ -181,6 +195,56 @@ def show(name: str):
db.close()
# ── annotate ─────────────────────────────────────────────────────────────────
@main.command()
@click.argument("draft_name")
@click.option("--note", "-n", default=None, help="Set/update the note text")
@click.option("--tag", "-t", multiple=True, help="Add a tag (can be used multiple times)")
@click.option("--remove-tag", "-r", multiple=True, help="Remove a tag (can be used multiple times)")
def annotate(draft_name: str, note: str | None, tag: tuple[str, ...], remove_tag: tuple[str, ...]):
"""Add or view annotations (notes & tags) for a draft."""
cfg = _get_config()
db = Database(cfg)
try:
draft = db.get_draft(draft_name)
if draft is None:
console.print(f"[red]Draft not found: {draft_name}[/]")
return
# If no options, display current annotation
if note is None and not tag and not remove_tag:
ann = db.get_annotation(draft_name)
if ann:
console.print(f"\n[bold]Annotation for {draft_name}[/]")
console.print(f" Note: {ann['note'] or '(empty)'}")
console.print(f" Tags: {', '.join(ann['tags']) if ann['tags'] else '(none)'}")
console.print(f" Updated: {ann['updated_at']}")
else:
console.print(f"[dim]No annotation for {draft_name}. Use --note or --tag to add one.[/]")
return
# Fetch existing tags for add/remove operations
existing = db.get_annotation(draft_name)
current_tags = existing["tags"] if existing else []
for t in tag:
if t not in current_tags:
current_tags.append(t)
for t in remove_tag:
if t in current_tags:
current_tags.remove(t)
db.upsert_annotation(draft_name, note=note, tags=current_tags)
ann = db.get_annotation(draft_name)
console.print(f"[green]Annotation updated for {draft_name}[/]")
console.print(f" Note: {ann['note'] or '(empty)'}")
console.print(f" Tags: {', '.join(ann['tags']) if ann['tags'] else '(none)'}")
finally:
db.close()
# ── analyze ──────────────────────────────────────────────────────────────────
@@ -188,7 +252,8 @@ def show(name: str):
@click.argument("name", required=False)
@click.option("--all", "analyze_all", is_flag=True, help="Analyze all unrated drafts")
@click.option("--limit", "-n", default=50, help="Max drafts to analyze (with --all)")
def analyze(name: str | None, analyze_all: bool, limit: int):
@click.option("--retry-failed", is_flag=True, help="Re-analyze drafts that previously failed (clears cache)")
def analyze(name: str | None, analyze_all: bool, limit: int, retry_failed: bool):
"""Analyze and rate drafts using Claude."""
from .analyzer import Analyzer
@@ -197,7 +262,29 @@ def analyze(name: str | None, analyze_all: bool, limit: int):
analyzer = Analyzer(cfg, db)
try:
if analyze_all:
if retry_failed:
# Find drafts that have cache entries but no ratings (failed analyses)
unrated = db.unrated_drafts(limit=limit)
retryable = []
for draft in unrated:
# Check if there's a cache entry for this draft (it was attempted)
row = db.conn.execute(
"SELECT COUNT(*) FROM llm_cache WHERE draft_name = ?",
(draft.name,),
).fetchone()
if row[0] > 0:
retryable.append(draft)
if not retryable:
console.print("No previously failed drafts to retry.")
else:
console.print(f"Retrying [bold]{len(retryable)}[/] previously failed drafts...")
count = 0
for draft in retryable:
rating = analyzer.rate_draft(draft.name, use_cache=False)
if rating:
count += 1
console.print(f"Successfully re-analyzed [bold green]{count}[/] of {len(retryable)} drafts")
elif analyze_all:
count = analyzer.rate_all_unrated(limit=limit)
console.print(f"Analyzed [bold green]{count}[/] drafts")
elif name:
@@ -217,6 +304,62 @@ def analyze(name: str | None, analyze_all: bool, limit: int):
db.close()
# ── ask ──────────────────────────────────────────────────────────────────────
@main.command()
@click.argument("question")
@click.option("--top", "-n", default=5, help="Number of source drafts to use")
@click.option("--cheap/--quality", default=True, help="Use Haiku (cheap) vs Sonnet (quality)")
def ask(question: str, top: int, cheap: bool):
"""Ask a natural language question about the drafts.
Examples:
ietf ask "Which drafts address agent authentication?"
ietf ask "What are the competing approaches to agent delegation?" --top 10
ietf ask "How do safety mechanisms work?" --cheap
"""
from .search import HybridSearch
cfg = _get_config()
db = Database(cfg)
try:
searcher = HybridSearch(cfg, db)
console.print(f"\n[dim]Searching for relevant drafts...[/]")
result = searcher.ask(question, top_k=top, cheap=cheap)
# Display the answer
console.print()
console.print("[bold cyan]Answer[/]")
console.print("[dim]" + "-" * 60 + "[/]")
console.print(result["answer"])
console.print()
# Display source drafts table
if result["sources"]:
table = Table(title="Source Drafts")
table.add_column("#", style="dim", width=3)
table.add_column("Draft", style="cyan", max_width=50)
table.add_column("Title", max_width=45)
table.add_column("Match", width=10)
table.add_column("Score", justify="right", width=8)
for i, src in enumerate(result["sources"], 1):
score_str = f"{src['similarity']:.3f}" if src.get("similarity") else "-"
table.add_row(
str(i),
src["name"],
src["title"][:45],
src.get("match_type", ""),
score_str,
)
console.print(table)
finally:
db.close()
# ── compare ──────────────────────────────────────────────────────────────────
@@ -232,7 +375,12 @@ def compare(names: tuple[str, ...]):
try:
result = analyzer.compare_drafts(list(names))
console.print(result)
if "error" in result:
console.print(f"[red]{result['error']}[/]")
else:
console.print(f"\n[bold cyan]Comparison of {len(result['drafts'])} drafts[/]")
console.print("[dim]" + "-" * 60 + "[/]")
console.print(result["text"])
finally:
db.close()
@@ -2107,7 +2255,8 @@ def draft_gen(gap_topic: str, output: str | None):
@main.command("config")
@click.option("--set", "set_key", nargs=2, help="Set a config key (e.g. --set claude_model claude-opus-4-20250514)")
def config_cmd(set_key: tuple[str, str] | None):
@click.option("--show", is_flag=True, help="Show effective config with env var sources noted")
def config_cmd(set_key: tuple[str, str] | None, show: bool):
"""Show or modify configuration."""
from dataclasses import asdict
cfg = _get_config()
@@ -2131,8 +2280,20 @@ def config_cmd(set_key: tuple[str, str] | None):
console.print(f"[red]Unknown config key: {key}[/]")
else:
from dataclasses import asdict
env_sources = cfg.env_sources()
for key, val in asdict(cfg).items():
console.print(f" [bold]{key}:[/] {val}")
source_note = ""
if key in env_sources:
source_note = f" [yellow](from ${env_sources[key]})[/]"
console.print(f" [bold]{key}:[/] {val}{source_note}")
if env_sources:
console.print(f"\n [dim]({len(env_sources)} value(s) overridden by environment variables)[/]")
# Note about ANTHROPIC_API_KEY
import os
if os.environ.get("ANTHROPIC_API_KEY"):
console.print(" [dim]ANTHROPIC_API_KEY is set in environment[/]")
else:
console.print(" [dim]ANTHROPIC_API_KEY is NOT set in environment[/]")
# ── pipeline ────────────────────────────────────────────────────────────────
@@ -2321,35 +2482,79 @@ def pipeline_quality(draft_id: int):
@pipeline.command("status")
def pipeline_status():
"""Show all generated drafts."""
"""Show pipeline health: processing stages, generated drafts, and API cost."""
cfg = _get_config()
db = Database(cfg)
try:
drafts = db.get_generated_drafts()
if not drafts:
console.print("No generated drafts yet. Run `ietf pipeline generate <topic>`")
return
# Pipeline health overview
total = db.count_drafts()
rated_count = len(db.drafts_with_ratings(limit=10000))
unrated = len(db.unrated_drafts(limit=10000))
unembedded = len(db.drafts_without_embeddings(limit=10000))
embedded_count = total - unembedded
no_ideas = len(db.drafts_without_ideas(limit=10000))
ideas_count = total - no_ideas
idea_total = db.idea_count()
gap_count = len(db.all_gaps())
input_tok, output_tok = db.total_tokens_used()
est_cost = (input_tok * 3.0 / 1_000_000) + (output_tok * 15.0 / 1_000_000)
table = Table(title=f"Generated Drafts ({len(drafts)})")
table.add_column("ID", justify="right", width=4)
table.add_column("Draft Name", style="cyan")
table.add_column("Gap Topic")
table.add_column("Family", width=15)
table.add_column("Status", width=10)
table.add_column("Quality", justify="right", width=7)
table.add_column("Created", width=10)
# Last update
snapshots = db.get_snapshots(limit=1)
last_update = snapshots[0]["snapshot_at"][:19] if snapshots else "never"
for d in drafts:
table.add_row(
str(d["id"]),
d["draft_name"],
d["gap_topic"][:30],
d.get("family_name", ""),
d.get("status", "?"),
f"{d.get('quality_score', 0):.1f}" if d.get("quality_score") else "-",
(d.get("created_at") or "")[:10],
)
console.print(table)
console.print("\n[bold]Pipeline Status[/]\n")
console.print(f" Total documents: [bold]{total}[/]")
console.print(f" Last update: {last_update}")
console.print()
# Stage table
stage_table = Table(title="Processing Stages")
stage_table.add_column("Stage", width=20)
stage_table.add_column("Done", justify="right", width=8)
stage_table.add_column("Missing", justify="right", width=8)
stage_table.add_column("Progress", width=20)
def bar(done, total_n):
pct = int(done / total_n * 100) if total_n > 0 else 0
filled = pct // 5
return f"[green]{'#' * filled}[/][dim]{'.' * (20 - filled)}[/] {pct}%"
stage_table.add_row("Rated", str(rated_count), str(unrated), bar(rated_count, total))
stage_table.add_row("Embedded", str(embedded_count), str(unembedded), bar(embedded_count, total))
stage_table.add_row("Ideas extracted", str(ideas_count), str(no_ideas), bar(ideas_count, total))
console.print(stage_table)
console.print(f"\n Total ideas: [bold]{idea_total}[/]")
console.print(f" Gaps identified: [bold]{gap_count}[/]")
console.print(f"\n API tokens: {input_tok:,} in + {output_tok:,} out")
console.print(f" Estimated cost: [bold]${est_cost:.2f}[/]")
# Generated drafts
gen_drafts = db.get_generated_drafts()
if gen_drafts:
console.print()
table = Table(title=f"Generated Drafts ({len(gen_drafts)})")
table.add_column("ID", justify="right", width=4)
table.add_column("Draft Name", style="cyan")
table.add_column("Gap Topic")
table.add_column("Family", width=15)
table.add_column("Status", width=10)
table.add_column("Quality", justify="right", width=7)
table.add_column("Created", width=10)
for d in gen_drafts:
table.add_row(
str(d["id"]),
d["draft_name"],
d["gap_topic"][:30],
d.get("family_name", ""),
d.get("status", "?"),
f"{d.get('quality_score', 0):.1f}" if d.get("quality_score") else "-",
(d.get("created_at") or "")[:10],
)
console.print(table)
finally:
db.close()
@@ -2397,28 +2602,38 @@ def observatory():
@observatory.command("update")
@click.option("--source", "-s", default=None, help="Comma-separated sources (e.g. ietf,w3c)")
@click.option("--full/--delta", default=False, help="Full refresh or delta only")
def observatory_update(source: str | None, full: bool):
@click.option("--dry-run", is_flag=True, default=False, help="Show what would happen without making changes")
def observatory_update(source: str | None, full: bool, dry_run: bool):
"""Fetch, analyze, and update the observatory."""
from .observatory import Observatory
from .analyzer import Analyzer
cfg = _get_config()
db = Database(cfg)
analyzer = Analyzer(cfg, db)
try:
obs = Observatory(cfg, db, analyzer)
if dry_run:
obs = Observatory(cfg, db)
else:
from .analyzer import Analyzer
analyzer = Analyzer(cfg, db)
obs = Observatory(cfg, db, analyzer)
sources = source.split(",") if source else None
console.print(f"[bold]Observatory update[/] ({'full' if full else 'delta'})")
result = obs.update(sources=sources, full=full)
mode = "full" if full else "delta"
console.print(f"[bold]Observatory update[/] ({mode}{' [DRY RUN]' if dry_run else ''})")
result = obs.update(sources=sources, full=full, dry_run=dry_run)
console.print(f"\n[bold green]Update complete![/]")
console.print(f" New docs: {result.get('new_docs', 0)}")
console.print(f" Analyzed: {result.get('analyzed', 0)}")
console.print(f" Embedded: {result.get('embedded', 0)}")
console.print(f" Ideas extracted: {result.get('ideas', 0)}")
if result.get("gaps_updated"):
console.print(f" Gaps re-analyzed: yes ({result.get('gap_count', 0)} gaps)")
if not dry_run:
console.print(f"\n[bold green]Update complete![/]")
console.print(f" New docs: {result.get('new_docs', 0)}")
console.print(f" Analyzed: {result.get('analyzed', 0)}")
console.print(f" Embedded: {result.get('embedded', 0)}")
console.print(f" Ideas extracted: {result.get('ideas', 0)}")
if result.get("gaps_changed"):
console.print(f" Gaps re-analyzed: yes")
if result.get("errors"):
console.print(f"\n [yellow]Errors ({len(result['errors'])}):[/]")
for err in result["errors"]:
console.print(f" - {err}")
finally:
db.close()
@@ -2676,3 +2891,105 @@ def monitor_status():
console.print(table)
finally:
db.close()
# ── export ──────────────────────────────────────────────────────────────────
@main.command()
@click.option("--type", "export_type", type=click.Choice(["drafts", "ideas", "gaps", "authors", "ratings"]),
required=True, help="Type of data to export")
@click.option("--format", "fmt", type=click.Choice(["json", "csv"]), default="json", help="Output format")
@click.option("--output", "-o", "output_file", type=click.Path(), default=None,
help="Output file (default: stdout)")
def export(export_type: str, fmt: str, output_file: str | None):
"""Export data as JSON or CSV."""
import csv as csv_mod
import io
import json
cfg = _get_config()
db = Database(cfg)
try:
rows: list[dict] = []
if export_type == "drafts":
drafts = db.list_drafts(limit=10000, order_by="name ASC")
for d in drafts:
rating = db.get_rating(d.name)
row = {
"name": d.name,
"title": d.title,
"rev": d.rev,
"date": d.date,
"pages": d.pages or 0,
"group": d.group or "",
}
if rating:
row["score"] = round(rating.composite_score, 2)
row["novelty"] = rating.novelty
row["maturity"] = rating.maturity
row["overlap"] = rating.overlap
row["momentum"] = rating.momentum
row["relevance"] = rating.relevance
row["categories"] = json.dumps(rating.categories)
row["summary"] = rating.summary
rows.append(row)
elif export_type == "ideas":
ideas = db.all_ideas()
rows = ideas
elif export_type == "gaps":
gaps = db.all_gaps()
rows = gaps
elif export_type == "authors":
top = db.top_authors(limit=10000)
for name, aff, cnt, drafts_list in top:
rows.append({
"name": name,
"affiliation": aff,
"draft_count": cnt,
"drafts": json.dumps(drafts_list),
})
elif export_type == "ratings":
pairs = db.drafts_with_ratings(limit=10000)
for draft, rating in pairs:
rows.append({
"name": draft.name,
"title": draft.title,
"score": round(rating.composite_score, 2),
"novelty": rating.novelty,
"maturity": rating.maturity,
"overlap": rating.overlap,
"momentum": rating.momentum,
"relevance": rating.relevance,
"categories": json.dumps(rating.categories),
"summary": rating.summary,
})
if fmt == "json":
text = json.dumps(rows, indent=2, ensure_ascii=False)
else:
# CSV
if not rows:
text = ""
else:
si = io.StringIO()
writer = csv_mod.DictWriter(si, fieldnames=rows[0].keys())
writer.writeheader()
for row in rows:
writer.writerow(row)
text = si.getvalue()
if output_file:
Path(output_file).write_text(text, encoding="utf-8")
console.print(f"Exported [bold green]{len(rows)}[/] {export_type} to [cyan]{output_file}[/] ({fmt})")
else:
click.echo(text)
finally:
db.close()