Christian Nennemann
757b781c67
Major features added by 5 parallel agent teams: - Semantic "Ask" (NL queries via FTS5 + embeddings + Claude synthesis) - Global search across drafts, ideas, authors, gaps - REST API expansion (14 endpoints, up from 3) with CSV/JSON export - Citation graph visualization (D3.js, 440 nodes, 2422 edges) - Standards readiness scoring (0-100 composite from 6 factors) - Side-by-side draft comparison view with shared/unique analysis - Annotation system (notes + tags per draft, DB-persisted) - Docker deployment (Dockerfile + docker-compose with Ollama) - Scheduled updates (cron script with log rotation) - Pipeline health dashboard (stage progress bars, cost tracking) - Test suite foundation (54 pytest tests covering DB, models, web data) Fixes: compare_drafts() stubbed→working, get_authors_for_draft() bug, source-aware analysis prompts, config env var overrides + validation, resilient batch error handling with --retry-failed, observatory --dry-run Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
586 lines
17 KiB
Python
586 lines
17 KiB
Python
"""IETF Draft Analyzer — Web Dashboard.
|
|
|
|
Run with: python src/webui/app.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Ensure project src is on path
|
|
_project_root = Path(__file__).resolve().parent.parent.parent
|
|
sys.path.insert(0, str(_project_root / "src"))
|
|
|
|
import csv
|
|
import io
|
|
import json
|
|
|
|
from flask import Flask, render_template, request, jsonify, abort, g, Response
|
|
|
|
from webui.data import (
|
|
get_db,
|
|
get_overview_stats,
|
|
get_category_counts,
|
|
get_drafts_page,
|
|
get_draft_detail,
|
|
get_rating_distributions,
|
|
get_timeline_data,
|
|
get_ideas_by_type,
|
|
get_all_gaps,
|
|
get_gap_detail,
|
|
get_generated_drafts,
|
|
read_generated_draft,
|
|
get_top_authors,
|
|
get_org_data,
|
|
get_category_radar_data,
|
|
get_score_histogram,
|
|
get_coauthor_network,
|
|
get_cross_org_data,
|
|
get_landscape_tsne,
|
|
get_similarity_graph,
|
|
get_timeline_animation_data,
|
|
get_idea_clusters,
|
|
get_monitor_status,
|
|
get_author_network_full,
|
|
get_citation_graph,
|
|
get_comparison_data,
|
|
get_ask_data,
|
|
global_search,
|
|
)
|
|
|
|
app = Flask(
|
|
__name__,
|
|
template_folder=str(Path(__file__).parent / "templates"),
|
|
static_folder=str(Path(__file__).parent / "static"),
|
|
static_url_path="/static",
|
|
)
|
|
app.config["SECRET_KEY"] = "ietf-dashboard-dev"
|
|
|
|
|
|
# --- Database lifecycle (per-request to avoid SQLite threading issues) ---
|
|
|
|
|
|
def db():
|
|
if "db" not in g:
|
|
g.db = get_db()
|
|
return g.db
|
|
|
|
|
|
@app.teardown_appcontext
|
|
def close_db(exception=None):
|
|
database = g.pop("db", None)
|
|
if database is not None:
|
|
database.close()
|
|
|
|
|
|
# --- Routes ---
|
|
|
|
|
|
@app.route("/")
|
|
def overview():
|
|
stats = get_overview_stats(db())
|
|
categories = get_category_counts(db())
|
|
timeline = get_timeline_data(db())
|
|
scores = get_score_histogram(db())
|
|
radar = get_category_radar_data(db())
|
|
return render_template(
|
|
"overview.html",
|
|
stats=stats,
|
|
categories=categories,
|
|
timeline=timeline,
|
|
scores=scores,
|
|
radar=radar,
|
|
)
|
|
|
|
|
|
@app.route("/drafts")
|
|
def drafts():
|
|
page = request.args.get("page", 1, type=int)
|
|
search = request.args.get("q", "")
|
|
category = request.args.get("cat", "")
|
|
source = request.args.get("source", "")
|
|
min_score = request.args.get("min_score", 0.0, type=float)
|
|
sort = request.args.get("sort", "score")
|
|
sort_dir = request.args.get("dir", "desc")
|
|
|
|
result = get_drafts_page(
|
|
db(),
|
|
page=page,
|
|
search=search,
|
|
category=category,
|
|
min_score=min_score,
|
|
sort=sort,
|
|
sort_dir=sort_dir,
|
|
source=source,
|
|
)
|
|
categories = get_category_counts(db())
|
|
return render_template(
|
|
"drafts.html",
|
|
result=result,
|
|
categories=categories,
|
|
search=search,
|
|
current_cat=category,
|
|
current_source=source,
|
|
min_score=min_score,
|
|
sort=sort,
|
|
sort_dir=sort_dir,
|
|
)
|
|
|
|
|
|
@app.route("/drafts/<path:name>")
|
|
def draft_detail(name: str):
|
|
database = db()
|
|
detail = get_draft_detail(database, name)
|
|
if not detail:
|
|
abort(404)
|
|
# Build set of draft ref IDs that exist in our DB for internal linking
|
|
ref_draft_ids = [r["id"] for r in detail.get("refs", []) if r["type"] == "draft"]
|
|
known_drafts = set()
|
|
if ref_draft_ids:
|
|
placeholders = ",".join("?" * len(ref_draft_ids))
|
|
rows = database.conn.execute(
|
|
f"SELECT name FROM drafts WHERE name IN ({placeholders})", ref_draft_ids
|
|
).fetchall()
|
|
known_drafts = {r["name"] for r in rows}
|
|
return render_template("draft_detail.html", draft=detail, known_drafts=known_drafts)
|
|
|
|
|
|
@app.route("/ideas")
|
|
def ideas():
|
|
data = get_ideas_by_type(db())
|
|
return render_template("ideas.html", data=data)
|
|
|
|
|
|
@app.route("/gaps")
|
|
def gaps():
|
|
gap_list = get_all_gaps(db())
|
|
generated = get_generated_drafts()
|
|
return render_template("gaps.html", gaps=gap_list, generated_drafts=generated)
|
|
|
|
|
|
@app.route("/gaps/demo")
|
|
def gaps_demo():
|
|
"""Show a pre-generated example draft so users can see output without API calls."""
|
|
generated = get_generated_drafts()
|
|
# Default to the first generated draft, or allow selection via query param
|
|
selected = request.args.get("file", "")
|
|
draft_text = None
|
|
draft_info = None
|
|
if selected:
|
|
draft_text = read_generated_draft(selected)
|
|
for gd in generated:
|
|
if gd["filename"] == selected:
|
|
draft_info = gd
|
|
break
|
|
elif generated:
|
|
draft_info = generated[0]
|
|
draft_text = read_generated_draft(draft_info["filename"])
|
|
return render_template(
|
|
"gap_demo.html",
|
|
generated_drafts=generated,
|
|
draft_text=draft_text,
|
|
draft_info=draft_info,
|
|
selected=selected,
|
|
)
|
|
|
|
|
|
@app.route("/gaps/<int:gap_id>")
|
|
def gap_detail(gap_id: int):
|
|
gap = get_gap_detail(db(), gap_id)
|
|
if not gap:
|
|
abort(404)
|
|
generated = get_generated_drafts()
|
|
return render_template("gap_detail.html", gap=gap, generated_drafts=generated)
|
|
|
|
|
|
@app.route("/gaps/<int:gap_id>/generate", methods=["POST"])
|
|
def gap_generate(gap_id: int):
|
|
"""Trigger draft generation for a gap. Returns JSON with the generated text."""
|
|
gap = get_gap_detail(db(), gap_id)
|
|
if not gap:
|
|
return jsonify({"error": "Gap not found"}), 404
|
|
|
|
try:
|
|
from ietf_analyzer.config import Config
|
|
from ietf_analyzer.analyzer import Analyzer
|
|
from ietf_analyzer.draftgen import DraftGenerator
|
|
|
|
cfg = Config.load()
|
|
database = db()
|
|
analyzer = Analyzer(cfg, database)
|
|
generator = DraftGenerator(cfg, database, analyzer)
|
|
|
|
# Generate into a file named after the gap
|
|
slug = gap["topic"].lower().replace(" ", "-")[:40]
|
|
output_path = str(Path(_project_root) / "data" / "reports" / "generated-drafts" / f"draft-gap-{gap_id}-{slug}.txt")
|
|
path = generator.generate(gap["topic"], output_path=output_path)
|
|
draft_text = Path(path).read_text(errors="replace")
|
|
|
|
return jsonify({
|
|
"success": True,
|
|
"text": draft_text,
|
|
"filename": Path(path).name,
|
|
"path": path,
|
|
})
|
|
except Exception as e:
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
|
|
@app.route("/ratings")
|
|
def ratings():
|
|
distributions = get_rating_distributions(db())
|
|
radar = get_category_radar_data(db())
|
|
return render_template(
|
|
"ratings.html",
|
|
dist=distributions,
|
|
radar=radar,
|
|
)
|
|
|
|
|
|
@app.route("/landscape")
|
|
def landscape():
|
|
distributions = get_rating_distributions(db())
|
|
tsne_data = get_landscape_tsne(db())
|
|
return render_template(
|
|
"landscape.html",
|
|
dist=distributions,
|
|
tsne_data=tsne_data,
|
|
)
|
|
|
|
|
|
@app.route("/timeline")
|
|
def timeline_animation():
|
|
data = get_timeline_animation_data(db())
|
|
return render_template("timeline.html", animation=data)
|
|
|
|
|
|
@app.route("/idea-clusters")
|
|
def idea_clusters():
|
|
data = get_idea_clusters(db())
|
|
return render_template("idea_clusters.html", clusters=data)
|
|
|
|
|
|
@app.route("/similarity")
|
|
def similarity():
|
|
network = get_similarity_graph(db())
|
|
return render_template("similarity.html", network=network)
|
|
|
|
|
|
@app.route("/authors")
|
|
def authors():
|
|
top = get_top_authors(db(), limit=50)
|
|
orgs = get_org_data(db(), limit=20)
|
|
network = get_author_network_full(db())
|
|
cross_org = get_cross_org_data(db(), limit=20)
|
|
return render_template(
|
|
"authors.html",
|
|
authors=top,
|
|
orgs=orgs,
|
|
orgs_data=orgs,
|
|
network=network,
|
|
cross_org=cross_org,
|
|
)
|
|
|
|
|
|
@app.route("/citations")
|
|
def citations():
|
|
graph = get_citation_graph(db())
|
|
return render_template("citations.html", graph=graph)
|
|
|
|
|
|
@app.route("/monitor")
|
|
def monitor_page():
|
|
status = get_monitor_status(db())
|
|
return render_template("monitor.html", status=status)
|
|
|
|
|
|
@app.route("/about")
|
|
def about():
|
|
stats = get_overview_stats(db())
|
|
return render_template("about.html", stats=stats)
|
|
|
|
|
|
@app.route("/impressum")
|
|
def impressum():
|
|
return render_template("impressum.html")
|
|
|
|
|
|
@app.route("/datenschutz")
|
|
def datenschutz():
|
|
return render_template("datenschutz.html")
|
|
|
|
|
|
@app.route("/search")
|
|
def search():
|
|
q = request.args.get("q", "").strip()
|
|
results = global_search(db(), q) if q else {"drafts": [], "ideas": [], "authors": [], "gaps": []}
|
|
total = sum(len(v) for v in results.values())
|
|
return render_template("search_results.html", query=q, results=results, total=total)
|
|
|
|
|
|
@app.route("/ask")
|
|
def ask_page():
|
|
question = request.args.get("q", "")
|
|
result = None
|
|
if question:
|
|
top_k = request.args.get("top", 5, type=int)
|
|
result = get_ask_data(db(), question, top_k=top_k)
|
|
return render_template("ask.html", question=question, result=result)
|
|
|
|
|
|
@app.route("/api/ask", methods=["POST"])
|
|
def api_ask():
|
|
"""Answer a question via hybrid search + Claude. Returns JSON."""
|
|
data = request.get_json(force=True, silent=True)
|
|
if not data or "question" not in data:
|
|
return jsonify({"error": "Missing 'question' in request body"}), 400
|
|
question = data["question"]
|
|
top_k = data.get("top_k", 5)
|
|
cheap = data.get("cheap", True)
|
|
result = get_ask_data(db(), question, top_k=top_k, cheap=cheap)
|
|
return jsonify(result)
|
|
|
|
|
|
@app.route("/compare")
|
|
def compare_page():
|
|
draft_names = request.args.get("drafts", "")
|
|
names = [n.strip() for n in draft_names.split(",") if n.strip()] if draft_names else []
|
|
data = None
|
|
if len(names) >= 2:
|
|
data = get_comparison_data(db(), names)
|
|
return render_template("comparison.html", names=names, data=data)
|
|
|
|
|
|
@app.route("/api/compare", methods=["POST"])
|
|
def api_compare():
|
|
"""Run Claude comparison for drafts. Returns JSON with comparison text."""
|
|
req_data = request.get_json(force=True, silent=True)
|
|
if not req_data or "drafts" not in req_data:
|
|
return jsonify({"error": "Missing 'drafts' in request body"}), 400
|
|
|
|
names = req_data["drafts"]
|
|
if len(names) < 2:
|
|
return jsonify({"error": "Need at least 2 drafts to compare"}), 400
|
|
|
|
try:
|
|
from ietf_analyzer.config import Config
|
|
from ietf_analyzer.analyzer import Analyzer
|
|
|
|
cfg = Config.load()
|
|
database = db()
|
|
analyzer = Analyzer(cfg, database)
|
|
result = analyzer.compare_drafts(names)
|
|
return jsonify(result)
|
|
except Exception as e:
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
|
|
# --- API endpoints for AJAX (used by client-side charts) ---
|
|
|
|
|
|
def _to_csv_response(rows: list[dict], filename: str = "export.csv") -> Response:
|
|
"""Convert a list of dicts to a CSV download response."""
|
|
if not rows:
|
|
return Response("", mimetype="text/csv",
|
|
headers={"Content-Disposition": f"attachment; filename={filename}"})
|
|
si = io.StringIO()
|
|
writer = csv.DictWriter(si, fieldnames=rows[0].keys())
|
|
writer.writeheader()
|
|
for row in rows:
|
|
# Flatten any list/dict values to JSON strings
|
|
flat = {}
|
|
for k, v in row.items():
|
|
if isinstance(v, (list, dict)):
|
|
flat[k] = json.dumps(v)
|
|
else:
|
|
flat[k] = v
|
|
writer.writerow(flat)
|
|
return Response(si.getvalue(), mimetype="text/csv",
|
|
headers={"Content-Disposition": f"attachment; filename={filename}"})
|
|
|
|
|
|
def _results_to_csv(results: dict) -> Response:
|
|
"""Convert global search results (multi-category) to a single CSV."""
|
|
rows = []
|
|
for category, items in results.items():
|
|
for item in items:
|
|
row = {"_category": category}
|
|
row.update(item)
|
|
rows.append(row)
|
|
return _to_csv_response(rows, "search_results.csv")
|
|
|
|
|
|
@app.route("/api/drafts")
|
|
def api_drafts():
|
|
page = request.args.get("page", 1, type=int)
|
|
search = request.args.get("q", "")
|
|
category = request.args.get("cat", "")
|
|
source = request.args.get("source", "")
|
|
min_score = request.args.get("min_score", 0.0, type=float)
|
|
sort = request.args.get("sort", "score")
|
|
sort_dir = request.args.get("dir", "desc")
|
|
data = get_drafts_page(db(), page=page, search=search, category=category,
|
|
min_score=min_score, sort=sort, sort_dir=sort_dir,
|
|
source=source)
|
|
if request.args.get("format") == "csv":
|
|
return _to_csv_response(data.get("drafts", []), "drafts.csv")
|
|
return jsonify(data)
|
|
|
|
|
|
@app.route("/api/stats")
|
|
def api_stats():
|
|
return jsonify(get_overview_stats(db()))
|
|
|
|
|
|
@app.route("/api/authors/network")
|
|
def api_author_network():
|
|
return jsonify(get_author_network_full(db()))
|
|
|
|
|
|
@app.route("/api/citations")
|
|
def api_citations():
|
|
min_refs = request.args.get("min_refs", 2, type=int)
|
|
return jsonify(get_citation_graph(db(), min_refs=min_refs))
|
|
|
|
|
|
@app.route("/api/search")
|
|
def api_search():
|
|
q = request.args.get("q", "").strip()
|
|
results = global_search(db(), q) if q else {"drafts": [], "ideas": [], "authors": [], "gaps": []}
|
|
if request.args.get("format") == "csv":
|
|
return _results_to_csv(results)
|
|
return jsonify(results)
|
|
|
|
|
|
@app.route("/api/ideas")
|
|
def api_ideas():
|
|
data = get_ideas_by_type(db())
|
|
if request.args.get("format") == "csv":
|
|
return _to_csv_response(data.get("ideas", []), "ideas.csv")
|
|
return jsonify(data)
|
|
|
|
|
|
@app.route("/api/gaps")
|
|
def api_gaps():
|
|
data = get_all_gaps(db())
|
|
if request.args.get("format") == "csv":
|
|
return _to_csv_response(data, "gaps.csv")
|
|
return jsonify(data)
|
|
|
|
|
|
@app.route("/api/gaps/<int:gap_id>")
|
|
def api_gap_detail(gap_id: int):
|
|
gap = get_gap_detail(db(), gap_id)
|
|
if not gap:
|
|
return jsonify({"error": "Gap not found"}), 404
|
|
return jsonify(gap)
|
|
|
|
|
|
@app.route("/api/ratings")
|
|
def api_ratings():
|
|
data = get_rating_distributions(db())
|
|
if request.args.get("format") == "csv":
|
|
# Transpose columnar data to rows
|
|
rows = []
|
|
for i in range(len(data.get("names", []))):
|
|
rows.append({
|
|
"name": data["names"][i],
|
|
"score": data["scores"][i],
|
|
"novelty": data["novelty"][i],
|
|
"maturity": data["maturity"][i],
|
|
"overlap": data["overlap"][i],
|
|
"momentum": data["momentum"][i],
|
|
"relevance": data["relevance"][i],
|
|
"category": data["categories"][i],
|
|
})
|
|
return _to_csv_response(rows, "ratings.csv")
|
|
return jsonify(data)
|
|
|
|
|
|
@app.route("/api/timeline")
|
|
def api_timeline():
|
|
data = get_timeline_data(db())
|
|
return jsonify(data)
|
|
|
|
|
|
@app.route("/api/landscape")
|
|
def api_landscape():
|
|
data = get_landscape_tsne(db())
|
|
if request.args.get("format") == "csv":
|
|
return _to_csv_response(data, "landscape.csv")
|
|
return jsonify(data)
|
|
|
|
|
|
@app.route("/api/similarity")
|
|
def api_similarity():
|
|
data = get_similarity_graph(db())
|
|
return jsonify(data)
|
|
|
|
|
|
@app.route("/api/idea-clusters")
|
|
def api_idea_clusters():
|
|
data = get_idea_clusters(db())
|
|
return jsonify(data)
|
|
|
|
|
|
@app.route("/api/monitor")
|
|
def api_monitor():
|
|
data = get_monitor_status(db())
|
|
return jsonify(data)
|
|
|
|
|
|
@app.route("/api/drafts/<path:name>")
|
|
def api_draft_detail(name: str):
|
|
detail = get_draft_detail(db(), name)
|
|
if not detail:
|
|
return jsonify({"error": "Draft not found"}), 404
|
|
return jsonify(detail)
|
|
|
|
|
|
@app.route("/api/categories")
|
|
def api_categories():
|
|
data = get_category_counts(db())
|
|
if request.args.get("format") == "csv":
|
|
rows = [{"category": k, "count": v} for k, v in data.items()]
|
|
return _to_csv_response(rows, "categories.csv")
|
|
return jsonify(data)
|
|
|
|
|
|
@app.route("/api/drafts/<path:name>/annotate", methods=["POST"])
|
|
def api_annotate(name: str):
|
|
"""Add or update annotation for a draft."""
|
|
import json as _json
|
|
database = db()
|
|
draft = database.get_draft(name)
|
|
if not draft:
|
|
return jsonify({"error": "Draft not found"}), 404
|
|
|
|
data = request.get_json(force=True, silent=True)
|
|
if not data:
|
|
return jsonify({"error": "Invalid JSON body"}), 400
|
|
|
|
note = data.get("note")
|
|
tags = data.get("tags")
|
|
add_tag = data.get("add_tag")
|
|
remove_tag = data.get("remove_tag")
|
|
|
|
# Handle add/remove tag operations
|
|
if add_tag or remove_tag:
|
|
existing = database.get_annotation(name)
|
|
current_tags = existing["tags"] if existing else []
|
|
if add_tag and add_tag not in current_tags:
|
|
current_tags.append(add_tag)
|
|
if remove_tag and remove_tag in current_tags:
|
|
current_tags.remove(remove_tag)
|
|
tags = current_tags
|
|
|
|
database.upsert_annotation(name, note=note, tags=tags)
|
|
annotation = database.get_annotation(name)
|
|
return jsonify({"success": True, "annotation": annotation})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print("Starting IETF Draft Analyzer Dashboard on http://127.0.0.1:5000")
|
|
app.run(debug=True, host="127.0.0.1", port=5000)
|