diff --git a/src/ietf_analyzer/db.py b/src/ietf_analyzer/db.py index 2a51111..7a5d9e9 100644 --- a/src/ietf_analyzer/db.py +++ b/src/ietf_analyzer/db.py @@ -639,10 +639,10 @@ class Database: def author_count(self) -> int: return self.conn.execute("SELECT COUNT(*) FROM authors").fetchone()[0] - def top_authors(self, limit: int = 20) -> list[tuple[str, str, int, list[str]]]: - """Return (name, affiliation, draft_count, [draft_names]).""" + def top_authors(self, limit: int = 20) -> list[tuple[str, str, int, list[str], int]]: + """Return (name, affiliation, draft_count, [draft_names], person_id).""" rows = self.conn.execute( - """SELECT a.name, a.affiliation, COUNT(da.draft_name) as cnt, + """SELECT a.person_id, a.name, a.affiliation, COUNT(da.draft_name) as cnt, GROUP_CONCAT(da.draft_name, '||') as drafts FROM authors a JOIN draft_authors da ON a.person_id = da.person_id @@ -653,10 +653,50 @@ class Database: ).fetchall() return [ (r["name"], r["affiliation"], r["cnt"], - r["drafts"].split("||") if r["drafts"] else []) + r["drafts"].split("||") if r["drafts"] else [], + r["person_id"]) for r in rows ] + def get_author_by_id(self, person_id: int) -> dict | None: + """Return author info by person_id, or None if not found.""" + row = self.conn.execute( + "SELECT * FROM authors WHERE person_id = ?", (person_id,) + ).fetchone() + if not row: + return None + return { + "person_id": row["person_id"], + "name": row["name"], + "ascii_name": row["ascii_name"], + "affiliation": row["affiliation"], + "resource_uri": row["resource_uri"], + } + + def get_author_drafts(self, person_id: int) -> list[str]: + """Return draft names for a given author.""" + rows = self.conn.execute( + "SELECT draft_name FROM draft_authors WHERE person_id = ? ORDER BY draft_name", + (person_id,), + ).fetchall() + return [r["draft_name"] for r in rows] + + def get_coauthors(self, person_id: int) -> list[dict]: + """Return co-authors for a given person (authors on the same drafts).""" + rows = self.conn.execute( + """SELECT DISTINCT a.person_id, a.name, a.affiliation, COUNT(DISTINCT da2.draft_name) as shared + FROM draft_authors da1 + JOIN draft_authors da2 ON da1.draft_name = da2.draft_name AND da2.person_id != da1.person_id + JOIN authors a ON da2.person_id = a.person_id + WHERE da1.person_id = ? + GROUP BY a.person_id + ORDER BY shared DESC""", + (person_id,), + ).fetchall() + return [{"person_id": r["person_id"], "name": r["name"], + "affiliation": r["affiliation"], "shared_drafts": r["shared"]} + for r in rows] + def top_orgs(self, limit: int = 20) -> list[tuple[str, int, int]]: """Return (org, author_count, draft_count).""" rows = self.conn.execute( diff --git a/src/ietf_analyzer/reports.py b/src/ietf_analyzer/reports.py index c8cc6ac..1f23559 100644 --- a/src/ietf_analyzer/reports.py +++ b/src/ietf_analyzer/reports.py @@ -433,7 +433,7 @@ class Reporter: "| # | Author | Organization | Drafts | Categories |", "|--:|--------|-------------|-------:|------------|", ]) - for rank, (name, aff, cnt, draft_names) in enumerate(top, 1): + for rank, (name, aff, cnt, draft_names, _pid) in enumerate(top, 1): cats: set[str] = set() for dn in draft_names: r = rating_map.get(dn) diff --git a/src/ietf_analyzer/visualize.py b/src/ietf_analyzer/visualize.py index 940a03b..96caad0 100644 --- a/src/ietf_analyzer/visualize.py +++ b/src/ietf_analyzer/visualize.py @@ -452,7 +452,7 @@ class Visualizer: # Get affiliations for coloring (normalized) from .orgs import normalize_org top_authors = self.db.top_authors(limit=200) - author_aff = {name: normalize_org(aff) for name, aff, _, _ in top_authors} + author_aff = {name: normalize_org(aff) for name, aff, _, _, _pid in top_authors} # Node sizing by degree degrees = dict(G.degree()) diff --git a/src/webui/blueprints/admin.py b/src/webui/blueprints/admin.py index de02bf7..534fb9d 100644 --- a/src/webui/blueprints/admin.py +++ b/src/webui/blueprints/admin.py @@ -17,6 +17,7 @@ from webui.data import ( get_rating_distributions, get_all_gaps, get_gap_detail, + get_drafts_for_gap, get_generated_drafts, read_generated_draft, get_monitor_status, @@ -30,6 +31,7 @@ from webui.data import ( get_citation_influence, get_bcp_analysis, get_idea_analysis, + get_idea_detail, get_trends_data, get_complexity_data, get_all_proposals, @@ -113,7 +115,9 @@ def gap_detail(gap_id: int): abort(404) generated = get_generated_drafts() gap_proposals = get_proposals_for_gap(db(), gap_id) - return render_template("gap_detail.html", gap=gap, generated_drafts=generated, proposals=gap_proposals) + related_drafts = get_drafts_for_gap(db(), gap_id) + return render_template("gap_detail.html", gap=gap, generated_drafts=generated, + proposals=gap_proposals, related_drafts=related_drafts) @admin_bp.route("/gaps//generate", methods=["POST"]) @@ -382,6 +386,17 @@ def api_idea_analysis(): return jsonify(data) +# ── Idea Detail ────────────────────────────────────────────────────────── + +@admin_bp.route("/ideas/") +@admin_required +def idea_detail(idea_id: int): + idea = get_idea_detail(db(), idea_id) + if not idea: + abort(404) + return render_template("idea_detail.html", idea=idea) + + # ── Trends & Complexity ────────────────────────────────────────────────── @admin_bp.route("/trends") diff --git a/src/webui/blueprints/pages.py b/src/webui/blueprints/pages.py index 45636d2..b16cc9b 100644 --- a/src/webui/blueprints/pages.py +++ b/src/webui/blueprints/pages.py @@ -27,6 +27,7 @@ from webui.data import ( get_ask_search, get_citation_influence, get_bcp_analysis, + get_author_detail, ) pages_bp = Blueprint("pages", __name__) @@ -144,6 +145,14 @@ def architecture(): return render_template("architecture.html", arch=data) +@pages_bp.route("/authors/") +def author_detail(person_id: int): + detail = get_author_detail(db(), person_id) + if not detail: + abort(404) + return render_template("author_detail.html", author=detail) + + @pages_bp.route("/authors") def authors(): top = get_top_authors(db(), limit=50) diff --git a/src/webui/data/__init__.py b/src/webui/data/__init__.py index 8aac5b3..2e7f5c1 100644 --- a/src/webui/data/__init__.py +++ b/src/webui/data/__init__.py @@ -37,6 +37,7 @@ from webui.data.authors import ( # noqa: F401 get_coauthor_network, get_cross_org_data, get_author_network_full, + get_author_detail, ) # Ratings @@ -51,6 +52,7 @@ from webui.data.ratings import ( # noqa: F401 from webui.data.gaps import ( # noqa: F401 get_all_gaps, get_gap_detail, + get_drafts_for_gap, ) # Analysis & Visualization @@ -74,6 +76,7 @@ from webui.data.analysis import ( # noqa: F401 get_comparison_data, get_architecture, get_idea_analysis, + get_idea_detail, get_trends_data, get_complexity_data, get_source_comparison, diff --git a/src/webui/data/analysis.py b/src/webui/data/analysis.py index 6c962b2..f78f672 100644 --- a/src/webui/data/analysis.py +++ b/src/webui/data/analysis.py @@ -103,6 +103,78 @@ def get_ideas_by_type(db: Database) -> dict: "ideas": all_ideas, } +def get_idea_detail(db: Database, idea_id: int) -> dict | None: + """Return a single idea with source draft info and similar ideas.""" + row = db.conn.execute("SELECT * FROM ideas WHERE id = ?", (idea_id,)).fetchone() + if not row: + return None + + idea = { + "id": row["id"], + "title": row["title"], + "description": row["description"], + "type": row["idea_type"], + "draft_name": row["draft_name"], + "novelty_score": row["novelty_score"], + } + + # Get source draft info + draft = db.get_draft(row["draft_name"]) + if draft: + idea["draft_title"] = draft.title + idea["draft_date"] = draft.date + + # Get category from ratings + rated = db.drafts_with_ratings(limit=2000) + for d, r in rated: + if d.name == row["draft_name"]: + idea["categories"] = r.categories + break + + # Find similar ideas using embeddings + similar = [] + emb_row = db.conn.execute( + "SELECT vector FROM idea_embeddings WHERE idea_id = ?", (idea_id,) + ).fetchone() + if emb_row: + target_vec = np.frombuffer(emb_row["vector"], dtype=np.float32) + all_embs = db.all_idea_embeddings() + # Compute cosine similarities + scores = [] + for other_id, other_vec in all_embs.items(): + if other_id == idea_id: + continue + cos_sim = float(np.dot(target_vec, other_vec) / ( + np.linalg.norm(target_vec) * np.linalg.norm(other_vec) + 1e-9)) + scores.append((other_id, cos_sim)) + scores.sort(key=lambda x: x[1], reverse=True) + top_5 = scores[:5] + + # Fetch idea details for top 5 + if top_5: + ids = [s[0] for s in top_5] + sim_map = {s[0]: s[1] for s in top_5} + placeholders = ",".join("?" * len(ids)) + sim_rows = db.conn.execute( + f"SELECT id, title, idea_type, draft_name FROM ideas WHERE id IN ({placeholders})", + ids, + ).fetchall() + sim_dict = {r["id"]: r for r in sim_rows} + for sid, score in top_5: + sr = sim_dict.get(sid) + if sr: + similar.append({ + "id": sr["id"], + "title": sr["title"], + "type": sr["idea_type"], + "draft_name": sr["draft_name"], + "similarity": round(score, 3), + }) + + idea["similar"] = similar + return idea + + def get_timeline_data(db: Database) -> TimelineData: """Return monthly counts by category for timeline chart.""" pairs = db.drafts_with_ratings(limit=1000) diff --git a/src/webui/data/authors.py b/src/webui/data/authors.py index b6268b9..6f9ced3 100644 --- a/src/webui/data/authors.py +++ b/src/webui/data/authors.py @@ -15,6 +15,7 @@ class AuthorInfo(TypedDict): affiliation: str draft_count: int drafts: list[str] + person_id: int class AuthorNetworkNode(TypedDict): """Node in the author network graph.""" @@ -50,8 +51,9 @@ def get_top_authors(db: Database, limit: int = 30) -> list[AuthorInfo]: """Return top authors by draft count.""" rows = db.top_authors(limit=limit) return [ - {"name": name, "affiliation": aff, "draft_count": cnt, "drafts": drafts} - for name, aff, cnt, drafts in rows + {"name": name, "affiliation": aff, "draft_count": cnt, "drafts": drafts, + "person_id": pid} + for name, aff, cnt, drafts, pid in rows ] def get_org_data(db: Database, limit: int = 20) -> list[dict]: @@ -71,7 +73,7 @@ def get_coauthor_network(db: Database, min_shared: int = 1) -> dict: top = db.top_authors(limit=100) # Build node set from authors who have co-authorships - author_info = {name: {"org": aff, "draft_count": cnt} for name, aff, cnt, _ in top} + author_info = {name: {"org": aff, "draft_count": cnt} for name, aff, cnt, _, _pid in top} node_set = set() edges = [] for a, b, shared in pairs: @@ -92,6 +94,49 @@ def get_coauthor_network(db: Database, min_shared: int = 1) -> dict: return {"nodes": nodes, "edges": edges} +def get_author_detail(db: Database, person_id: int) -> dict | None: + """Return author detail with drafts, ratings, and co-authors.""" + author = db.get_author_by_id(person_id) + if not author: + return None + + draft_names = db.get_author_drafts(person_id) + drafts_map = db.get_drafts_by_names(draft_names) + + # Get ratings for each draft + rated = db.drafts_with_ratings(limit=2000) + rating_map = {d.name: r for d, r in rated} + + drafts = [] + for dn in draft_names: + d = drafts_map.get(dn) + if not d: + continue + r = rating_map.get(dn) + drafts.append({ + "name": d.name, + "title": d.title, + "date": d.date, + "status": d.status, + "categories": r.categories if r else [], + "score": round(r.composite_score, 2) if r else None, + "novelty": r.novelty if r else None, + "maturity": r.maturity if r else None, + "relevance": r.relevance if r else None, + }) + + coauthors = db.get_coauthors(person_id) + + return { + "person_id": author["person_id"], + "name": author["name"], + "affiliation": author["affiliation"], + "ascii_name": author["ascii_name"], + "drafts": drafts, + "coauthors": coauthors, + } + + def get_cross_org_data(db: Database, limit: int = 20) -> list[dict]: """Return cross-org collaboration pairs.""" rows = db.cross_org_collaborations(limit=limit) @@ -122,7 +167,7 @@ def _compute_author_network_full(db: Database) -> AuthorNetwork: # Author info map author_info = {} - for name, aff, cnt, drafts in top: + for name, aff, cnt, drafts, _pid in top: scores = [draft_score[dn] for dn in drafts if dn in draft_score] avg = round(sum(scores) / len(scores), 2) if scores else 0 author_info[name] = { diff --git a/src/webui/data/drafts.py b/src/webui/data/drafts.py index a5d9e45..e3efde3 100644 --- a/src/webui/data/drafts.py +++ b/src/webui/data/drafts.py @@ -94,7 +94,7 @@ def get_category_summary(db: Database, category: str) -> dict | None: # Author lookup: draft_name -> [author names] author_drafts_map: dict[str, list[str]] = defaultdict(list) - for name, aff, cnt, drafts in all_authors: + for name, aff, cnt, drafts, *_ in all_authors: for dn in drafts: author_drafts_map[dn].append(name) @@ -116,7 +116,7 @@ def get_category_summary(db: Database, category: str) -> dict | None: author_counter: Counter = Counter() org_counter: Counter = Counter() author_aff: dict[str, str] = {} - for name, aff, cnt, drafts in all_authors: + for name, aff, cnt, drafts, *_ in all_authors: author_aff[name] = aff or "" for d, r in cat_pairs: for a in author_drafts_map.get(d.name, []): diff --git a/src/webui/data/gaps.py b/src/webui/data/gaps.py index 83b9b05..e84aa33 100644 --- a/src/webui/data/gaps.py +++ b/src/webui/data/gaps.py @@ -1,6 +1,8 @@ """Gap analysis data access functions.""" from __future__ import annotations +import re + from ietf_analyzer.db import Database @@ -18,3 +20,68 @@ def get_gap_detail(db: Database, gap_id: int) -> dict | None: if g["id"] == gap_id: return g return None + +def get_drafts_for_gap(db: Database, gap_id: int) -> list[dict]: + """Find drafts related to a gap by searching evidence for draft names + and searching draft titles/abstracts for gap topic keywords.""" + gap = get_gap_detail(db, gap_id) + if not gap: + return [] + + found_names: set[str] = set() + + # 1. Extract draft names mentioned in evidence text + evidence = gap.get("evidence", "") or "" + # Match draft-xxx-yyy-zzz patterns + draft_refs = re.findall(r'draft-[\w-]+', evidence) + found_names.update(draft_refs) + + # 2. Search drafts by gap topic keywords + topic = gap.get("topic", "") + # Extract meaningful keywords (3+ chars, skip common words) + stopwords = {"the", "and", "for", "with", "from", "that", "this", "are", "was", + "not", "but", "have", "has", "had", "will", "can", "all", "each", + "which", "their", "been", "into", "more", "other", "some", "than", + "may", "its", "also", "between", "should", "would", "could", "does"} + words = [w.lower() for w in re.findall(r'[A-Za-z]{3,}', topic) if w.lower() not in stopwords] + + # Search for drafts matching topic keywords + if words: + # Use the most specific keywords (longer words first) + keywords = sorted(words, key=len, reverse=True)[:3] + for kw in keywords: + like = f"%{kw}%" + rows = db.conn.execute( + """SELECT name FROM drafts + WHERE title LIKE ? OR abstract LIKE ? + LIMIT 20""", + (like, like), + ).fetchall() + for r in rows: + found_names.add(r["name"]) + + if not found_names: + return [] + + # Fetch draft details + ratings + drafts_map = db.get_drafts_by_names(list(found_names)) + rated = db.drafts_with_ratings(limit=2000) + rating_map = {d.name: r for d, r in rated} + + results = [] + for name in sorted(found_names): + d = drafts_map.get(name) + if not d: + continue + r = rating_map.get(name) + results.append({ + "name": d.name, + "title": d.title, + "date": d.date, + "score": round(r.composite_score, 2) if r else None, + "categories": r.categories if r else [], + }) + + # Sort by score descending + results.sort(key=lambda x: x.get("score") or 0, reverse=True) + return results diff --git a/src/webui/obsidian_export.py b/src/webui/obsidian_export.py index d0917aa..befec86 100644 --- a/src/webui/obsidian_export.py +++ b/src/webui/obsidian_export.py @@ -115,7 +115,7 @@ def build_obsidian_vault(db: Database) -> bytes: # Author info by draft author_drafts: dict[str, list[str]] = defaultdict(list) author_info: dict[str, dict] = {} - for name, aff, cnt, drafts in all_authors: + for name, aff, cnt, drafts, *_ in all_authors: author_info[name] = {"affiliation": aff or "", "draft_count": cnt, "drafts": drafts} for dn in drafts: author_drafts[dn].append(name) @@ -269,11 +269,11 @@ generated: {date.today().isoformat()} f"**{len(all_authors)}** authors contributing to AI/agent Internet-Drafts.\n\n", "| Author | Affiliation | Drafts |\n|---|---|---|\n", ] - for name, aff, cnt, drafts in sorted(all_authors, key=lambda x: x[2], reverse=True): + for name, aff, cnt, drafts, *_ in sorted(all_authors, key=lambda x: x[2], reverse=True): author_index_lines.append(f"| [[{name}]] | {aff or ''} | {cnt} |\n") zf.writestr(f"{prefix}/Authors/index.md", "".join(author_index_lines)) - for name, aff, cnt, drafts in all_authors: + for name, aff, cnt, drafts, *_ in all_authors: fm = f"---\ntags: [author]\naffiliation: \"{aff or ''}\"\ndraft_count: {cnt}\n---\n" body = f"\n# {name}\n\n" if aff: diff --git a/src/webui/templates/author_detail.html b/src/webui/templates/author_detail.html new file mode 100644 index 0000000..556b791 --- /dev/null +++ b/src/webui/templates/author_detail.html @@ -0,0 +1,111 @@ +{% extends "base.html" %} +{% set active_page = "authors" %} + +{% block title %}{{ author.name }} — IETF Draft Analyzer{% endblock %} + +{% block content %} + +
+ + + + + Back to Authors + +
+ + +
+
+
+ {{ author.name[0]|upper if author.name else '?' }} +
+
+

{{ author.name }}

+ {% if author.affiliation %} +

{{ author.affiliation }}

+ {% endif %} +
+ {{ author.drafts|length }} draft{{ 's' if author.drafts|length != 1 }} + {{ author.coauthors|length }} co-author{{ 's' if author.coauthors|length != 1 }} +
+
+
+
+ +
+ +
+
+

+ + Authored Drafts ({{ author.drafts|length }}) +

+
+ {% for d in author.drafts %} + +
+
+

{{ d.title }}

+
+ {{ d.name }} + {% if d.date %} + {{ d.date }} + {% endif %} +
+ {% if d.categories %} +
+ {% for cat in d.categories[:3] %} + {{ cat }} + {% endfor %} +
+ {% endif %} +
+ {% if d.score is not none %} + + {{ d.score }} + + {% endif %} +
+ + {% endfor %} + {% if not author.drafts %} +

No drafts found.

+ {% endif %} +
+
+
+ + +
+
+

+ + Co-Authors ({{ author.coauthors|length }}) +

+
    + {% for ca in author.coauthors %} +
  • +
    + {{ ca.name[0]|upper if ca.name else '?' }} +
    +
    + {{ ca.name }} + {% if ca.affiliation %} +
    {{ ca.affiliation }}
    + {% endif %} +
    {{ ca.shared_drafts }} shared draft{{ 's' if ca.shared_drafts != 1 }}
    +
    +
    • + {% endfor %} + {% if not author.coauthors %} +

    No co-authors found.

    + {% endif %} +
+
+
+
+{% endblock %} diff --git a/src/webui/templates/authors.html b/src/webui/templates/authors.html index 9af62b8..d869021 100644 --- a/src/webui/templates/authors.html +++ b/src/webui/templates/authors.html @@ -211,7 +211,7 @@ {{ loop.index }} - {{ a.name }} + {{ a.name }} {{ a.affiliation }} diff --git a/src/webui/templates/draft_detail.html b/src/webui/templates/draft_detail.html index 71173ba..fe81700 100644 --- a/src/webui/templates/draft_detail.html +++ b/src/webui/templates/draft_detail.html @@ -331,7 +331,7 @@ {{ a.name[0]|upper if a.name else '?' }}
- {{ a.name }} + {{ a.name }} {% if a.affiliation %}
{{ a.affiliation }}
{% endif %} diff --git a/src/webui/templates/gap_detail.html b/src/webui/templates/gap_detail.html index ee03d44..bca987d 100644 --- a/src/webui/templates/gap_detail.html +++ b/src/webui/templates/gap_detail.html @@ -86,6 +86,51 @@
+ +{% if related_drafts %} +
+

+ + Related Drafts ({{ related_drafts|length }}) +

+
+ {% for d in related_drafts[:15] %} + +
+
+

{{ d.title }}

+
+ {{ d.name }} + {% if d.date %} + {{ d.date }} + {% endif %} +
+ {% if d.categories %} +
+ {% for cat in d.categories[:3] %} + {{ cat }} + {% endfor %} +
+ {% endif %} +
+ {% if d.score is not none %} + + {{ d.score }} + + {% endif %} +
+ + {% endfor %} +
+ {% if related_drafts|length > 15 %} +

Showing 15 of {{ related_drafts|length }} related drafts.

+ {% endif %} +
+{% endif %} +
diff --git a/src/webui/templates/idea_detail.html b/src/webui/templates/idea_detail.html new file mode 100644 index 0000000..5686f6e --- /dev/null +++ b/src/webui/templates/idea_detail.html @@ -0,0 +1,123 @@ +{% extends "base.html" %} +{% set active_page = "ideas" %} + +{% block title %}{{ idea.title }} — Idea Detail{% endblock %} + +{% block extra_head %} + +{% endblock %} + +{% block content %} + + + + +
+
+

{{ idea.title }}

+
+ {% if idea.type %} + {% set type_lower = idea.type|lower %} + + {{ idea.type }} + + {% endif %} + {% if idea.novelty_score is not none and idea.novelty_score %} + N:{{ idea.novelty_score }} + {% endif %} +
+
+ + {% if idea.description %} +
+

Description

+

{{ idea.description }}

+
+ {% endif %} + + +
+

Source Draft

+ + {{ idea.draft_title or idea.draft_name }} + + {{ idea.draft_name }} + {% if idea.draft_date %} + {{ idea.draft_date }} + {% endif %} + {% if idea.categories %} +
+ {% for cat in idea.categories[:3] %} + {{ cat }} + {% endfor %} +
+ {% endif %} +
+
+ + +{% if idea.similar %} +
+

+ + Most Similar Ideas (by embedding cosine similarity) +

+
+ {% for sim in idea.similar %} + +
+
+

{{ sim.title }}

+
+ {{ sim.draft_name }} + {% if sim.type %} + {% set st = sim.type|lower %} + + {{ sim.type }} + + {% endif %} +
+
+ {{ sim.similarity }} +
+ + {% endfor %} +
+
+{% else %} +
+

Similar Ideas

+

No embeddings available for this idea. Run the embedding pipeline to enable similarity search.

+
+{% endif %} +{% endblock %}