Idea quality pipeline, web UI features, academic paper

- Tighten idea extraction prompts (1-4 ideas, no sub-features) reducing 1,907 ideas to 468 across 434 drafts (78% reduction) - Add embedding-based dedup (ietf dedup-ideas) for same-draft similarity - Add novelty scoring (ietf ideas score) and filtering (ietf ideas filter) using Claude to rate ideas 1-5, removing 49 generic building blocks - Final count: 419 high-quality ideas (avg 1.1/draft) - Web UI: gap explorer with live draft generation and pre-generated demos - Web UI: D3.js author collaboration network (498 nodes, 1142 edges, 68 clusters, org filtering, interactive zoom/pan) - Academic paper: 15-page LaTeX workshop paper analyzing the 434-draft AI agent standards landscape - Save improvement ideas backlog to data/reports/improvement-ideas.md Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-06 22:17:57 +01:00
parent 3c3d7e649f
commit 6e3a387778
29 changed files with 6575 additions and 240 deletions
--- a/src/webui/templates/overview.html
+++ b/src/webui/templates/overview.html
@@ -0,0 +1,205 @@
+{% extends "base.html" %}
+{% set active_page = "overview" %}
+
+{% block title %}Overview — IETF Draft Analyzer{% endblock %}
+
+{% block content %}
+<div class="mb-8">
+    <h1 class="text-2xl font-bold text-white">Dashboard Overview</h1>
+    <p class="text-slate-400 text-sm mt-1">IETF AI/Agent Internet-Drafts at a glance</p>
+</div>
+
+<!-- Stat cards -->
+<div class="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-5 gap-4 mb-8">
+    <a href="/drafts" class="stat-card rounded-xl border border-slate-800 p-5 relative overflow-hidden hover:border-blue-500/40 transition group">
+        <div class="absolute top-0 left-0 w-full h-1 bg-gradient-to-r from-blue-500 to-blue-400"></div>
+        <div class="text-3xl font-bold text-blue-400">{{ stats.total_drafts }}</div>
+        <div class="text-xs text-slate-400 mt-1 uppercase tracking-wider group-hover:text-blue-400/70 transition">Total Drafts &rarr;</div>
+    </a>
+    <a href="/ratings" class="stat-card rounded-xl border border-slate-800 p-5 relative overflow-hidden hover:border-emerald-500/40 transition group">
+        <div class="absolute top-0 left-0 w-full h-1 bg-gradient-to-r from-emerald-500 to-emerald-400"></div>
+        <div class="text-3xl font-bold text-emerald-400">{{ stats.rated_count }}</div>
+        <div class="text-xs text-slate-400 mt-1 uppercase tracking-wider group-hover:text-emerald-400/70 transition">Rated Drafts &rarr;</div>
+    </a>
+    <a href="/authors" class="stat-card rounded-xl border border-slate-800 p-5 relative overflow-hidden hover:border-purple-500/40 transition group">
+        <div class="absolute top-0 left-0 w-full h-1 bg-gradient-to-r from-purple-500 to-purple-400"></div>
+        <div class="text-3xl font-bold text-purple-400">{{ stats.author_count }}</div>
+        <div class="text-xs text-slate-400 mt-1 uppercase tracking-wider group-hover:text-purple-400/70 transition">Authors &rarr;</div>
+    </a>
+    <a href="/ideas" class="stat-card rounded-xl border border-slate-800 p-5 relative overflow-hidden hover:border-amber-500/40 transition group">
+        <div class="absolute top-0 left-0 w-full h-1 bg-gradient-to-r from-amber-500 to-amber-400"></div>
+        <div class="text-3xl font-bold text-amber-400">{{ stats.idea_count }}</div>
+        <div class="text-xs text-slate-400 mt-1 uppercase tracking-wider group-hover:text-amber-400/70 transition">Ideas &rarr;</div>
+    </a>
+    <a href="/gaps" class="stat-card rounded-xl border border-slate-800 p-5 relative overflow-hidden hover:border-red-500/40 transition group">
+        <div class="absolute top-0 left-0 w-full h-1 bg-gradient-to-r from-red-500 to-red-400"></div>
+        <div class="text-3xl font-bold text-red-400">{{ stats.gap_count }}</div>
+        <div class="text-xs text-slate-400 mt-1 uppercase tracking-wider group-hover:text-red-400/70 transition">Gaps Found &rarr;</div>
+    </a>
+</div>
+
+<!-- Charts row 1: Score distribution + Category donut -->
+<div class="grid grid-cols-1 lg:grid-cols-2 gap-6 mb-6">
+    <div class="bg-slate-900 rounded-xl border border-slate-800 p-5">
+        <h2 class="text-sm font-semibold text-slate-300 mb-3">Composite Score Distribution</h2>
+        <div id="scoreHist" style="height: 300px;"></div>
+    </div>
+    <div class="bg-slate-900 rounded-xl border border-slate-800 p-5">
+        <h2 class="text-sm font-semibold text-slate-300 mb-3">Drafts by Category</h2>
+        <div id="categoryPie" style="height: 300px;"></div>
+    </div>
+</div>
+
+<!-- Timeline (full width) -->
+<div class="bg-slate-900 rounded-xl border border-slate-800 p-5 mb-6">
+    <h2 class="text-sm font-semibold text-slate-300 mb-3">Submissions Over Time</h2>
+    <div id="timeline" style="height: 350px;"></div>
+</div>
+
+<!-- Category radar -->
+<div class="bg-slate-900 rounded-xl border border-slate-800 p-5">
+    <h2 class="text-sm font-semibold text-slate-300 mb-3">Category Rating Profiles</h2>
+    <div id="radar" style="height: 420px;"></div>
+</div>
+{% endblock %}
+
+{% block extra_scripts %}
+<script>
+// Shared Plotly config
+const PLOTLY_LAYOUT = {
+    paper_bgcolor: 'transparent',
+    plot_bgcolor: 'transparent',
+    font: { color: '#94a3b8', family: 'Inter, system-ui, sans-serif', size: 12 },
+    margin: { t: 30, r: 20, b: 40, l: 40 },
+    xaxis: { gridcolor: '#1e293b', zerolinecolor: '#334155' },
+    yaxis: { gridcolor: '#1e293b', zerolinecolor: '#334155' },
+};
+const PLOTLY_CONFIG = { responsive: true, displayModeBar: false };
+
+const PALETTE = [
+    '#3b82f6', '#ef4444', '#22c55e', '#a855f7', '#f59e0b',
+    '#06b6d4', '#ec4899', '#84cc16', '#f97316', '#8b5cf6',
+    '#14b8a6', '#e11d48', '#64748b', '#eab308', '#6366f1',
+];
+
+// --- Score histogram ---
+const scores = {{ scores | tojson }};
+if (scores.length > 0) {
+    Plotly.newPlot('scoreHist', [{
+        x: scores,
+        type: 'histogram',
+        nbinsx: 20,
+        marker: {
+            color: 'rgba(59, 130, 246, 0.7)',
+            line: { color: '#3b82f6', width: 1 },
+        },
+        hovertemplate: 'Score: %{x}<br>Count: %{y}<extra></extra>',
+    }], {
+        ...PLOTLY_LAYOUT,
+        xaxis: { ...PLOTLY_LAYOUT.xaxis, title: { text: 'Composite Score', font: { size: 11 } } },
+        yaxis: { ...PLOTLY_LAYOUT.yaxis, title: { text: 'Count', font: { size: 11 } } },
+    }, PLOTLY_CONFIG);
+} else {
+    document.getElementById('scoreHist').innerHTML = '<p class="text-slate-500 text-sm text-center mt-20">No score data available</p>';
+}
+
+// --- Category donut ---
+const categories = {{ categories | tojson }};
+const catNames = Object.keys(categories);
+const catVals = Object.values(categories);
+if (catNames.length > 0) {
+    Plotly.newPlot('categoryPie', [{
+        labels: catNames,
+        values: catVals,
+        type: 'pie',
+        hole: 0.45,
+        textinfo: 'label+percent',
+        textposition: 'outside',
+        textfont: { size: 10, color: '#94a3b8' },
+        hovertemplate: '%{label}<br>%{value} drafts (%{percent})<extra></extra>',
+        marker: { colors: PALETTE },
+        pull: catVals.map((_, i) => i === 0 ? 0.03 : 0),
+    }], {
+        ...PLOTLY_LAYOUT,
+        showlegend: false,
+        margin: { t: 10, r: 10, b: 10, l: 10 },
+    }, PLOTLY_CONFIG);
+    // Click category to filter drafts
+    document.getElementById('categoryPie').on('plotly_click', function(data) {
+        const cat = data.points[0].label;
+        if (cat) window.location.href = '/drafts?cat=' + encodeURIComponent(cat);
+    });
+} else {
+    document.getElementById('categoryPie').innerHTML = '<p class="text-slate-500 text-sm text-center mt-20">No category data available</p>';
+}
+
+// --- Timeline (stacked area) ---
+const timeline = {{ timeline | tojson }};
+if (timeline.months && timeline.months.length > 0) {
+    const timeTraces = timeline.categories.map((cat, i) => ({
+        x: timeline.months,
+        y: timeline.series[cat],
+        name: cat,
+        type: 'scatter',
+        mode: 'lines',
+        stackgroup: 'one',
+        line: { width: 0.5, color: PALETTE[i % PALETTE.length] },
+        fillcolor: PALETTE[i % PALETTE.length] + '80',
+        hovertemplate: '%{x}<br>' + cat + ': %{y}<extra></extra>',
+    }));
+    Plotly.newPlot('timeline', timeTraces, {
+        ...PLOTLY_LAYOUT,
+        xaxis: { ...PLOTLY_LAYOUT.xaxis, title: { text: 'Month', font: { size: 11 } } },
+        yaxis: { ...PLOTLY_LAYOUT.yaxis, title: { text: 'Drafts', font: { size: 11 } } },
+        legend: { font: { size: 10, color: '#94a3b8' }, orientation: 'h', y: -0.2, x: 0.5, xanchor: 'center' },
+        hovermode: 'x unified',
+    }, PLOTLY_CONFIG);
+} else {
+    document.getElementById('timeline').innerHTML = '<p class="text-slate-500 text-sm text-center mt-20">No timeline data available</p>';
+}
+
+// --- Category radar ---
+const radar = {{ radar | tojson }};
+const dims = ['novelty', 'maturity', 'relevance', 'momentum', 'low_overlap'];
+const dimLabels = ['Novelty', 'Maturity', 'Relevance', 'Momentum', 'Low Overlap'];
+const radarCats = Object.keys(radar);
+if (radarCats.length > 0) {
+    const radarTraces = radarCats.map((cat, i) => {
+        const vals = radar[cat];
+        return {
+            type: 'scatterpolar',
+            r: dims.map(d => vals[d]).concat([vals[dims[0]]]),
+            theta: dimLabels.concat([dimLabels[0]]),
+            fill: 'toself',
+            fillcolor: PALETTE[i % PALETTE.length] + '20',
+            line: { color: PALETTE[i % PALETTE.length], width: 2 },
+            name: cat + ' (' + vals.count + ')',
+            opacity: 0.85,
+            hovertemplate: cat + '<br>%{theta}: %{r:.1f}<extra></extra>',
+        };
+    });
+    Plotly.newPlot('radar', radarTraces, {
+        ...PLOTLY_LAYOUT,
+        polar: {
+            bgcolor: 'transparent',
+            radialaxis: {
+                visible: true,
+                range: [0, 5],
+                gridcolor: '#1e293b',
+                color: '#64748b',
+                tickfont: { size: 10 },
+            },
+            angularaxis: {
+                gridcolor: '#1e293b',
+                color: '#94a3b8',
+                tickfont: { size: 11 },
+            },
+        },
+        legend: { font: { size: 10, color: '#94a3b8' }, x: 1.05, y: 0.5 },
+        margin: { t: 30, r: 120, b: 30, l: 60 },
+    }, PLOTLY_CONFIG);
+} else {
+    document.getElementById('radar').innerHTML = '<p class="text-slate-500 text-sm text-center mt-20">No radar data available</p>';
+}
+</script>
+{% endblock %}