Idea quality pipeline, web UI features, academic paper

- Tighten idea extraction prompts (1-4 ideas, no sub-features) reducing
  1,907 ideas to 468 across 434 drafts (78% reduction)
- Add embedding-based dedup (ietf dedup-ideas) for same-draft similarity
- Add novelty scoring (ietf ideas score) and filtering (ietf ideas filter)
  using Claude to rate ideas 1-5, removing 49 generic building blocks
- Final count: 419 high-quality ideas (avg 1.1/draft)
- Web UI: gap explorer with live draft generation and pre-generated demos
- Web UI: D3.js author collaboration network (498 nodes, 1142 edges,
  68 clusters, org filtering, interactive zoom/pan)
- Academic paper: 15-page LaTeX workshop paper analyzing the 434-draft
  AI agent standards landscape
- Save improvement ideas backlog to data/reports/improvement-ideas.md

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-06 22:17:57 +01:00
parent 3c3d7e649f
commit 6e3a387778
29 changed files with 6575 additions and 240 deletions

View File

@@ -0,0 +1,211 @@
{% extends "base.html" %}
{% set active_page = "ratings" %}
{% block title %}Ratings — IETF Draft Analyzer{% endblock %}
{% block content %}
<div class="mb-6">
<h1 class="text-2xl font-bold text-white">Rating Analytics</h1>
<p class="text-slate-400 text-sm mt-1">Distribution and analysis of AI-generated ratings</p>
</div>
<!-- Score Distribution -->
<div class="bg-slate-900 rounded-xl border border-slate-800 p-5 mb-6">
<h2 class="text-sm font-semibold text-slate-300 mb-3">Composite Score Distribution</h2>
<div id="scoreHist" style="height: 300px;"></div>
</div>
<div class="grid grid-cols-1 lg:grid-cols-2 gap-6 mb-6">
<!-- Dimension Box Plots -->
<div class="bg-slate-900 rounded-xl border border-slate-800 p-5">
<h2 class="text-sm font-semibold text-slate-300 mb-3">Score Distributions by Dimension</h2>
<div id="dimDist" style="height: 350px;"></div>
</div>
<!-- Category Radar -->
<div class="bg-slate-900 rounded-xl border border-slate-800 p-5">
<h2 class="text-sm font-semibold text-slate-300 mb-3">Category Rating Profiles</h2>
<div id="radar" style="height: 350px;"></div>
</div>
</div>
<!-- Scatter: novelty vs maturity -->
<div class="bg-slate-900 rounded-xl border border-slate-800 p-5 mb-6">
<h2 class="text-sm font-semibold text-slate-300 mb-3">Novelty vs Maturity (bubble = relevance)</h2>
<div id="scatter" style="height: 450px;"></div>
</div>
<!-- Top 20 Leaderboard -->
<div class="bg-slate-900 rounded-xl border border-slate-800 overflow-hidden">
<div class="p-4 border-b border-slate-800">
<h2 class="text-sm font-semibold text-slate-300">Top 20 Drafts by Composite Score</h2>
</div>
<div class="overflow-x-auto">
<table class="w-full text-sm">
<thead>
<tr class="border-b border-slate-800 text-left text-xs text-slate-500">
<th class="px-4 py-3 font-medium">#</th>
<th class="px-4 py-3 font-medium">Draft</th>
<th class="px-4 py-3 font-medium text-center">Score</th>
<th class="px-4 py-3 font-medium text-center">Novelty</th>
<th class="px-4 py-3 font-medium text-center">Maturity</th>
<th class="px-4 py-3 font-medium text-center">Relevance</th>
<th class="px-4 py-3 font-medium text-center">Momentum</th>
<th class="px-4 py-3 font-medium text-center">Overlap</th>
<th class="px-4 py-3 font-medium">Category</th>
</tr>
</thead>
<tbody id="leaderboard" class="divide-y divide-slate-800/50">
</tbody>
</table>
</div>
</div>
{% endblock %}
{% block extra_scripts %}
<script>
const PLOTLY_LAYOUT = {
paper_bgcolor: 'rgba(0,0,0,0)', plot_bgcolor: 'rgba(0,0,0,0)',
font: { color: '#94a3b8', family: 'Inter, system-ui, sans-serif', size: 12 },
margin: { t: 20, r: 20, b: 40, l: 50 },
xaxis: { gridcolor: '#1e293b', zerolinecolor: '#334155' },
yaxis: { gridcolor: '#1e293b', zerolinecolor: '#334155' },
};
const CFG = { responsive: true, displayModeBar: false };
const dist = {{ dist | tojson }};
const radar = {{ radar | tojson }};
// Score Histogram
Plotly.newPlot('scoreHist', [{
x: dist.scores,
type: 'histogram',
nbinsx: 25,
marker: { color: '#3b82f6', line: { color: '#1e40af', width: 1 } },
hovertemplate: 'Score: %{x:.1f}<br>Count: %{y}<extra></extra>',
}], {
...PLOTLY_LAYOUT,
xaxis: { ...PLOTLY_LAYOUT.xaxis, title: 'Composite Score' },
yaxis: { ...PLOTLY_LAYOUT.yaxis, title: 'Count' },
}, CFG);
// Box plots for each dimension
const dims = ['novelty', 'maturity', 'overlap', 'momentum', 'relevance'];
const dimLabelsBox = ['Novelty', 'Maturity', 'Overlap', 'Momentum', 'Relevance'];
const colors = ['#3b82f6', '#22c55e', '#ef4444', '#f59e0b', '#a855f7'];
const boxTraces = dims.map((d, i) => ({
y: dist[d], name: dimLabelsBox[i],
type: 'box', marker: { color: colors[i] }, boxmean: true,
}));
Plotly.newPlot('dimDist', boxTraces, {
...PLOTLY_LAYOUT,
showlegend: false,
yaxis: { ...PLOTLY_LAYOUT.yaxis, range: [0.5, 5.5], dtick: 1 },
}, CFG);
// Radar
const radarDims = ['novelty', 'maturity', 'relevance', 'momentum', 'low_overlap'];
const radarLabels = ['Novelty', 'Maturity', 'Relevance', 'Momentum', 'Low Overlap'];
const radarTraces = Object.entries(radar).map(([cat, vals]) => ({
type: 'scatterpolar',
r: radarDims.map(d => vals[d]).concat([vals[radarDims[0]]]),
theta: radarLabels.concat([radarLabels[0]]),
fill: 'toself', name: `${cat} (${vals.count})`, opacity: 0.4,
}));
Plotly.newPlot('radar', radarTraces, {
...PLOTLY_LAYOUT,
polar: {
bgcolor: 'rgba(0,0,0,0)',
radialaxis: { visible: true, range: [0, 5], gridcolor: '#1e293b', color: '#64748b' },
angularaxis: { gridcolor: '#1e293b', color: '#94a3b8' },
},
legend: { font: { size: 10, color: '#94a3b8' } },
margin: { t: 30, r: 60, b: 30, l: 60 },
}, CFG);
// Scatter: novelty vs maturity
const catGroups = {};
dist.names.forEach((name, i) => {
const cat = dist.categories[i];
if (!catGroups[cat]) catGroups[cat] = { x: [], y: [], size: [], text: [] };
catGroups[cat].x.push(dist.novelty[i] + (Math.random() - 0.5) * 0.3);
catGroups[cat].y.push(dist.maturity[i] + (Math.random() - 0.5) * 0.3);
catGroups[cat].size.push(Math.max(dist.relevance[i] * 4, 6));
catGroups[cat].text.push(name);
});
const scatterTraces = Object.entries(catGroups).map(([cat, d]) => ({
x: d.x, y: d.y, text: d.text, name: cat,
mode: 'markers', type: 'scatter',
marker: { size: d.size, opacity: 0.7 },
hovertemplate: '<b>%{text}</b><br>Novelty: %{x:.1f}<br>Maturity: %{y:.1f}<extra>' + cat + '</extra>',
}));
Plotly.newPlot('scatter', scatterTraces, {
...PLOTLY_LAYOUT,
xaxis: { ...PLOTLY_LAYOUT.xaxis, title: 'Novelty', range: [0.5, 5.5], dtick: 1 },
yaxis: { ...PLOTLY_LAYOUT.yaxis, title: 'Maturity', range: [0.5, 5.5], dtick: 1 },
legend: { font: { size: 10, color: '#94a3b8' } },
hovermode: 'closest',
}, CFG);
// Click scatter points to navigate to draft detail
document.getElementById('scatter').on('plotly_click', function(data) {
const pt = data.points[0];
if (pt.text) {
window.location.href = '/drafts/' + pt.text;
}
});
// Top 20 Leaderboard
(function buildLeaderboard() {
// Combine arrays into objects and sort by score descending
const drafts = dist.names.map((name, i) => ({
name,
score: dist.scores[i],
novelty: dist.novelty[i],
maturity: dist.maturity[i],
relevance: dist.relevance[i],
momentum: dist.momentum[i],
overlap: dist.overlap[i],
category: dist.categories[i],
}));
drafts.sort((a, b) => b.score - a.score);
const tbody = document.getElementById('leaderboard');
const top20 = drafts.slice(0, 20);
function scoreClass(score) {
if (score >= 3.5) return 'score-high';
if (score >= 2.5) return 'score-mid';
return 'score-low';
}
function dimBadge(val) {
const cls = val >= 4 ? 'text-green-400' : val >= 3 ? 'text-yellow-400' : 'text-slate-500';
return `<span class="${cls}">${val}</span>`;
}
top20.forEach((d, i) => {
const shortName = d.name.replace('draft-', '').substring(0, 40);
const row = document.createElement('tr');
row.className = 'hover:bg-slate-800/50 transition';
row.innerHTML = `
<td class="px-4 py-3 text-slate-500 font-mono text-xs">${i + 1}</td>
<td class="px-4 py-3">
<a href="/drafts/${d.name}" class="text-blue-400 hover:text-blue-300 transition text-xs font-mono">${shortName}</a>
</td>
<td class="px-4 py-3 text-center">
<span class="score-badge ${scoreClass(d.score)}">${d.score.toFixed(2)}</span>
</td>
<td class="px-4 py-3 text-center">${dimBadge(d.novelty)}</td>
<td class="px-4 py-3 text-center">${dimBadge(d.maturity)}</td>
<td class="px-4 py-3 text-center">${dimBadge(d.relevance)}</td>
<td class="px-4 py-3 text-center">${dimBadge(d.momentum)}</td>
<td class="px-4 py-3 text-center">${dimBadge(d.overlap)}</td>
<td class="px-4 py-3">
<span class="px-2 py-0.5 rounded text-[10px] bg-slate-800 text-slate-400">${d.category}</span>
</td>
`;
tbody.appendChild(row);
});
})();
</script>
{% endblock %}