feat: refresh pipeline, reorganize draft generation, polish public pages

Pipeline refresh: - Extract ideas from 46 remaining drafts (844 total ideas now) - Clear stale llm_cache entries blocking re-extraction - Re-run gap analysis with expanded corpus: 10 gaps (was 12), fresh IDs #1-#10 - Re-link 3 proposals to new gap IDs - Add scripts/pipeline-refresh.sh for reproducible runs Draft generation moved from gaps to proposals: - Remove "Generate Internet-Draft" section from gap detail page - Add it to proposal detail page instead (proposals → generate I-D flow) - New route POST /proposals/<id>/generate with richer context (proposal title + description + linked gap topics) - Remove misleading "Search related drafts" link from gap page (related drafts already shown inline) Public page polish: - Overview: update subtitle to mention all 6 standards bodies - About: describe multi-source scope (IETF, ISO, ITU-T, ETSI, NIST, W3C) - About: add guiding question ("Where is the AI agent standards race heading?") - Obsidian export button hidden in production mode (prior commit) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-09 05:39:13 +01:00
parent f8ed2b83e9
commit 42b4546ded
7 changed files with 215 additions and 159 deletions
--- a/data/drafts.db
+++ b/data/drafts.db
--- a/scripts/pipeline-refresh.sh
+++ b/scripts/pipeline-refresh.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Pipeline refresh: extract missing ideas, re-score, then re-run gap analysis
+# Run from project root: bash scripts/pipeline-refresh.sh
+
+set -euo pipefail
+cd "$(dirname "$0")/.."
+export PYTHONPATH=src
+
+echo "=== Step 1/3: Extract ideas from 38 remaining drafts ==="
+python -m ietf_analyzer.cli ideas --all --cheap --batch 5 --limit 50
+
+echo ""
+echo "=== Step 2/3: Score new ideas for novelty ==="
+python -m ietf_analyzer.cli ideas score --cheap --batch 10
+
+echo ""
+echo "=== Step 3/3: Re-run gap analysis ==="
+python -m ietf_analyzer.cli gaps --refresh
+
+echo ""
+echo "=== Done ==="
+python -c "
+from ietf_analyzer.config import Config
+from ietf_analyzer.db import Database
+cfg = Config.load()
+db = Database(cfg)
+ideas = db.conn.execute('SELECT count(*) FROM ideas').fetchone()[0]
+gaps = db.conn.execute('SELECT count(*) FROM gaps').fetchone()[0]
+print(f'Ideas: {ideas}, Gaps: {gaps}')
+"
--- a/src/webui/blueprints/admin.py
+++ b/src/webui/blueprints/admin.py
@@ -119,6 +119,7 @@ def gap_detail(gap_id: int):
    generated = get_generated_drafts()
    gap_proposals = get_proposals_for_gap(db(), gap_id)
    related_drafts = get_drafts_for_gap(db(), gap_id)
+
    return render_template("gap_detail.html", gap=gap, generated_drafts=generated,
                           proposals=gap_proposals, related_drafts=related_drafts)

@@ -514,6 +515,53 @@ def proposal_delete(proposal_id):
    return redirect(url_for("admin.proposals"))


+@admin_bp.route("/proposals/<int:proposal_id>/generate", methods=["POST"])
+@admin_required
+def proposal_generate(proposal_id):
+    """Generate an Internet-Draft from a proposal and its linked gaps."""
+    proposal = get_proposal_detail(db(), proposal_id)
+    if not proposal:
+        return jsonify({"error": "Proposal not found"}), 404
+
+    try:
+        from ietf_analyzer.config import Config
+        from ietf_analyzer.analyzer import Analyzer
+        from ietf_analyzer.draftgen import DraftGenerator
+
+        cfg = Config.load()
+        database = db()
+        analyzer = Analyzer(cfg, database)
+        generator = DraftGenerator(cfg, database, analyzer)
+
+        # Build rich topic from proposal + linked gaps
+        topic = proposal["title"]
+        gap_context = ""
+        if proposal.get("gaps"):
+            gap_topics = [g["topic"] for g in proposal["gaps"]]
+            gap_context = "\n\nThis proposal addresses these gaps:\n" + "\n".join(
+                f"- {g['topic']}: {g['description'][:200]}" for g in proposal["gaps"]
+            )
+        if proposal.get("description"):
+            gap_context += f"\n\nProposal description: {proposal['description']}"
+
+        slug = proposal.get("slug", "proposal")[:40]
+        output_path = str(
+            Path(_project_root) / "data" / "reports" / "generated-drafts"
+            / f"draft-proposal-{proposal_id}-{slug}.txt"
+        )
+        path = generator.generate(topic + gap_context, output_path=output_path)
+        draft_text = Path(path).read_text(errors="replace")
+
+        return jsonify({
+            "success": True,
+            "text": draft_text,
+            "filename": Path(path).name,
+            "path": path,
+        })
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
@admin_bp.route("/api/proposals")
@admin_required
 def api_proposals():
--- a/src/webui/templates/about.html
+++ b/src/webui/templates/about.html
@@ -10,14 +10,21 @@
    <div class="bg-slate-900 rounded-xl border border-slate-800 p-6 mb-6">
        <h2 class="text-lg font-semibold text-white mb-3">What is this?</h2>
        <p class="text-sm text-slate-400 leading-relaxed mb-4">
-            A tool for tracking, categorizing, rating, and mapping IETF Internet-Drafts
-            focused on AI and agent-related topics. It uses Claude for analysis and rating,
-            Ollama for embeddings, and SQLite for storage.
+            A research tool for tracking, categorizing, rating, and mapping standardization
+            documents on AI and agent-related topics across six standards bodies:
+            <span class="text-slate-200">IETF</span>,
+            <span class="text-slate-200">ISO/IEC</span>,
+            <span class="text-slate-200">ITU-T</span>,
+            <span class="text-slate-200">ETSI</span>,
+            <span class="text-slate-200">NIST</span>, and
+            <span class="text-slate-200">W3C</span>.
+            It uses Claude for analysis and rating, Ollama for embeddings, and SQLite for storage.
        </p>
        <p class="text-sm text-slate-400 leading-relaxed">
-            The dashboard provides interactive visualizations of the draft landscape,
+            The dashboard provides interactive visualizations of the standardization landscape,
            including category breakdowns, rating distributions, author networks,
-            extracted ideas, and gap analysis.
+            extracted ideas, and gap analysis — answering the question:
+            <em class="text-slate-300">Where is the AI agent standards race heading, and what's missing?</em>
        </p>
    </div>

@@ -54,10 +61,12 @@
    <div class="bg-slate-900 rounded-xl border border-slate-800 p-6 mb-6">
        <h2 class="text-lg font-semibold text-white mb-3">Data Collection Methodology</h2>
        <p class="text-sm text-slate-400 leading-relaxed mb-4">
-            Drafts are discovered by searching the
+            IETF drafts are discovered via the
            <a href="https://datatracker.ietf.org" class="text-blue-400 hover:text-blue-300 transition">IETF Datatracker API</a>
-            for documents whose abstract contains any of the following keywords.
-            Only drafts submitted since <span class="text-slate-200 font-medium">{{ fetch_since }}</span> are included.
+            by searching abstracts for the keywords below
+            (only drafts since <span class="text-slate-200 font-medium">{{ fetch_since }}</span>).
+            ISO, ITU-T, ETSI, NIST, and W3C documents are sourced from their respective public catalogs
+            using related search terms.
        </p>

        <h3 class="text-sm font-semibold text-slate-300 mb-2">Search Keywords</h3>
@@ -124,7 +133,7 @@
            <li><span class="text-slate-200 font-medium">Embeddings:</span> Ollama (nomic-embed-text)</li>
            <li><span class="text-slate-200 font-medium">Storage:</span> SQLite with FTS5 full-text search</li>
            <li><span class="text-slate-200 font-medium">Dashboard:</span> Flask, Tailwind CSS, Plotly.js</li>
-            <li><span class="text-slate-200 font-medium">Data source:</span> IETF Datatracker API</li>
+            <li><span class="text-slate-200 font-medium">Data sources:</span> IETF Datatracker, ISO, ITU-T, ETSI, NIST, W3C</li>
        </ul>
    </div>
 </div>
--- a/src/webui/templates/gap_detail.html
+++ b/src/webui/templates/gap_detail.html
@@ -3,27 +3,7 @@

 {% block title %}{{ gap.topic }} — Gap Explorer{% endblock %}

-{% block extra_head %}
-<style>
-    .draft-output {
-        max-height: 70vh;
-        overflow-y: auto;
-    }
-    .draft-output::-webkit-scrollbar { width: 6px; }
-    .draft-output::-webkit-scrollbar-track { background: #0f172a; }
-    .draft-output::-webkit-scrollbar-thumb { background: #334155; border-radius: 3px; }
-    .generating-spinner {
-        display: inline-block;
-        width: 1rem;
-        height: 1rem;
-        border: 2px solid rgba(255,255,255,0.3);
-        border-top-color: white;
-        border-radius: 50%;
-        animation: spin 0.8s linear infinite;
-    }
-    @keyframes spin { to { transform: rotate(360deg); } }
-</style>
-{% endblock %}
+{% block extra_head %}{% endblock %}

 {% block content %}
 <!-- Breadcrumb -->
@@ -71,19 +51,14 @@
    </div>

    <!-- Links -->
-    <div class="mt-4 pt-4 border-t border-slate-800/50 flex flex-wrap gap-3">
-        <a href="/drafts?q={{ gap.topic | urlencode }}" class="inline-flex items-center gap-1.5 text-xs text-blue-400/70 hover:text-blue-400 transition">
-            <svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z"/></svg>
-            Search related drafts
-        </a>
    {% if gap.category %}
-        <span class="text-slate-700">|</span>
+    <div class="mt-4 pt-4 border-t border-slate-800/50">
        <a href="/drafts?cat={{ gap.category | urlencode }}" class="inline-flex items-center gap-1.5 text-xs text-blue-400/70 hover:text-blue-400 transition">
            <svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"/></svg>
            Browse {{ gap.category }} drafts
        </a>
-        {% endif %}
    </div>
+    {% endif %}
 </div>

 <!-- Related Drafts -->
@@ -169,126 +144,16 @@
    {% endif %}
 </div>

-<!-- Draft Generation Section -->
+<!-- Pre-generated examples -->
+{% if generated_drafts %}
 <div class="bg-slate-900 rounded-xl border border-slate-800 p-6">
-    <div class="flex items-center justify-between mb-4">
-        <div>
-            <h2 class="text-lg font-semibold text-white">Generate Internet-Draft</h2>
-            <p class="text-xs text-slate-500 mt-1">Use AI to generate a full Internet-Draft addressing this gap</p>
-        </div>
-        <button id="generateBtn" onclick="generateDraft({{ gap.id }})"
-            class="inline-flex items-center gap-2 px-4 py-2 bg-blue-600 hover:bg-blue-500 disabled:bg-slate-700 disabled:text-slate-500 text-white text-sm font-medium rounded-lg transition">
-            <svg id="genIcon" class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z"/></svg>
-            <span id="genText">Generate Draft</span>
-        </button>
-    </div>
-
-    <!-- Status area -->
-    <div id="genStatus" class="hidden mb-4 p-3 rounded-lg bg-blue-500/10 border border-blue-500/20">
-        <div class="flex items-center gap-2 text-sm text-blue-400">
-            <span class="generating-spinner"></span>
-            <span id="statusText">Generating draft... This may take 1-2 minutes.</span>
-        </div>
-    </div>
-
-    <!-- Error area -->
-    <div id="genError" class="hidden mb-4 p-3 rounded-lg bg-red-500/10 border border-red-500/20">
-        <p class="text-sm text-red-400" id="errorText"></p>
-    </div>
-
-    <!-- Generated draft output -->
-    <div id="draftOutput" class="hidden">
-        <div class="flex items-center justify-between mb-3">
-            <h3 class="text-sm font-semibold text-slate-300">Generated Draft</h3>
-            <button onclick="downloadDraft()" class="inline-flex items-center gap-1.5 px-3 py-1.5 bg-slate-800 hover:bg-slate-700 text-slate-300 text-xs font-medium rounded-lg transition">
-                <svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 10v6m0 0l-3-3m3 3l3-3m2 8H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"/></svg>
-                Download .txt
-            </button>
-        </div>
-        <div class="draft-output bg-slate-950 rounded-lg border border-slate-800 p-4">
-            <pre id="draftText" class="text-xs text-slate-300 font-mono leading-relaxed whitespace-pre-wrap"></pre>
-        </div>
-    </div>
-
-    <!-- Hint to demo -->
-    <div id="demoHint" class="mt-4 text-xs text-slate-600">
-        Want to see what generated drafts look like without waiting?
-        <a href="/gaps/demo" class="text-blue-500 hover:text-blue-400 transition">View the demo page</a>
-        with {{ generated_drafts | length }} pre-generated examples.
-    </div>
+    <h2 class="text-lg font-semibold text-white mb-2">Generated Drafts</h2>
+    <p class="text-xs text-slate-500 mb-4">
+        {{ generated_drafts | length }} pre-generated example{{ 's' if generated_drafts | length != 1 }}.
+        <a href="/gaps/demo" class="text-blue-400 hover:text-blue-300 transition">View all</a>
+    </p>
 </div>
+{% endif %}
 {% endblock %}

-{% block extra_scripts %}
-<script>
-let generatedText = '';
-let generatedFilename = '';
-
-function generateDraft(gapId) {
-    const btn = document.getElementById('generateBtn');
-    const genIcon = document.getElementById('genIcon');
-    const genText = document.getElementById('genText');
-    const status = document.getElementById('genStatus');
-    const error = document.getElementById('genError');
-    const output = document.getElementById('draftOutput');
-    const hint = document.getElementById('demoHint');
-
-    // Disable button, show spinner
-    btn.disabled = true;
-    genIcon.innerHTML = '';
-    genIcon.classList.add('generating-spinner');
-    genText.textContent = 'Generating...';
-    status.classList.remove('hidden');
-    error.classList.add('hidden');
-    output.classList.add('hidden');
-    hint.classList.add('hidden');
-
-    fetch(`/gaps/${gapId}/generate`, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-    })
-    .then(r => r.json())
-    .then(data => {
-        status.classList.add('hidden');
-        if (data.error) {
-            error.classList.remove('hidden');
-            document.getElementById('errorText').textContent = data.error;
-            btn.disabled = false;
-            genIcon.classList.remove('generating-spinner');
-            genIcon.innerHTML = '<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z"/>';
-            genText.textContent = 'Retry';
-        } else {
-            generatedText = data.text;
-            generatedFilename = data.filename || 'generated-draft.txt';
-            document.getElementById('draftText').textContent = data.text;
-            output.classList.remove('hidden');
-            genIcon.classList.remove('generating-spinner');
-            genIcon.innerHTML = '<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 13l4 4L19 7"/>';
-            genText.textContent = 'Done';
-        }
-    })
-    .catch(err => {
-        status.classList.add('hidden');
-        error.classList.remove('hidden');
-        document.getElementById('errorText').textContent = 'Network error: ' + err.message;
-        btn.disabled = false;
-        genIcon.classList.remove('generating-spinner');
-        genIcon.innerHTML = '<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z"/>';
-        genText.textContent = 'Retry';
-    });
-}
-
-function downloadDraft() {
-    if (!generatedText) return;
-    const blob = new Blob([generatedText], { type: 'text/plain' });
-    const url = URL.createObjectURL(blob);
-    const a = document.createElement('a');
-    a.href = url;
-    a.download = generatedFilename;
-    document.body.appendChild(a);
-    a.click();
-    document.body.removeChild(a);
-    URL.revokeObjectURL(url);
-}
-</script>
-{% endblock %}
+{% block extra_scripts %}{% endblock %}
--- a/src/webui/templates/overview.html
+++ b/src/webui/templates/overview.html
@@ -9,7 +9,7 @@
 <div class="mb-8 flex flex-wrap items-start justify-between gap-4">
    <div>
        <h1 class="text-2xl font-bold text-white">Dashboard Overview</h1>
-        <p class="text-slate-400 text-sm mt-1">IETF AI/Agent Internet-Drafts at a glance. Drafts are fetched from the IETF Datatracker, then analyzed by Claude AI across five dimensions (novelty, maturity, overlap, momentum, relevance) to produce a composite score from 1.0 to 5.0.</p>
+        <p class="text-slate-400 text-sm mt-1">AI/Agent standardization landscape at a glance. 761 documents from IETF, ISO, ITU-T, ETSI, NIST, and W3C — analyzed by Claude AI across five dimensions (novelty, maturity, overlap, momentum, relevance) to produce a composite score from 1.0 to 5.0.</p>
    </div>
    {% if is_admin %}
    <a href="/export/obsidian"
--- a/src/webui/templates/proposal_detail.html
+++ b/src/webui/templates/proposal_detail.html
@@ -109,11 +109,115 @@

 <!-- Content -->
 {% if proposal.content_md %}
-<div class="bg-slate-900 rounded-xl border border-slate-800 p-6">
+<div class="bg-slate-900 rounded-xl border border-slate-800 p-6 mb-6">
    <h2 class="text-lg font-semibold text-white mb-4">Content</h2>
    <div class="bg-slate-950 rounded-lg border border-slate-800 p-4">
        <pre class="text-sm text-slate-300 font-mono leading-relaxed whitespace-pre-wrap">{{ proposal.content_md }}</pre>
    </div>
 </div>
 {% endif %}
+
+<!-- Generate Internet-Draft -->
+<div class="bg-slate-900 rounded-xl border border-slate-800 p-6">
+    <div class="flex items-center justify-between mb-4">
+        <div>
+            <h2 class="text-lg font-semibold text-white">Generate Internet-Draft</h2>
+            <p class="text-xs text-slate-500 mt-1">Use AI to generate a full IETF Internet-Draft from this proposal{% if proposal.gaps %} and its {{ proposal.gaps | length }} linked gap{{ 's' if proposal.gaps | length != 1 }}{% endif %}</p>
+        </div>
+        <button id="generateBtn" onclick="generateDraft({{ proposal.id }})"
+            class="inline-flex items-center gap-2 px-4 py-2 bg-blue-600 hover:bg-blue-500 disabled:bg-slate-700 disabled:text-slate-500 text-white text-sm font-medium rounded-lg transition">
+            <svg id="genIcon" class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z"/></svg>
+            <span id="genText">Generate Draft</span>
+        </button>
+    </div>
+
+    <!-- Status area -->
+    <div id="genStatus" class="hidden mb-4 p-3 rounded-lg bg-blue-500/10 border border-blue-500/20">
+        <div class="flex items-center gap-2 text-sm text-blue-400">
+            <span class="inline-block w-4 h-4 border-2 border-white/30 border-t-white rounded-full animate-spin"></span>
+            <span>Generating draft... This may take 1-2 minutes.</span>
+        </div>
+    </div>
+
+    <!-- Error area -->
+    <div id="genError" class="hidden mb-4 p-3 rounded-lg bg-red-500/10 border border-red-500/20">
+        <p class="text-sm text-red-400" id="errorText"></p>
+    </div>
+
+    <!-- Generated draft output -->
+    <div id="draftOutput" class="hidden">
+        <div class="flex items-center justify-between mb-3">
+            <h3 class="text-sm font-semibold text-slate-300">Generated Draft</h3>
+            <button onclick="downloadDraft()" class="inline-flex items-center gap-1.5 px-3 py-1.5 bg-slate-800 hover:bg-slate-700 text-slate-300 text-xs font-medium rounded-lg transition">
+                <svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 10v6m0 0l-3-3m3 3l3-3m2 8H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"/></svg>
+                Download .txt
+            </button>
+        </div>
+        <div class="max-h-[70vh] overflow-y-auto bg-slate-950 rounded-lg border border-slate-800 p-4">
+            <pre id="draftText" class="text-xs text-slate-300 font-mono leading-relaxed whitespace-pre-wrap"></pre>
+        </div>
+    </div>
+</div>
+{% endblock %}
+
+{% block extra_scripts %}
+<script>
+let generatedText = '';
+let generatedFilename = '';
+
+function generateDraft(proposalId) {
+    const btn = document.getElementById('generateBtn');
+    const genText = document.getElementById('genText');
+    const status = document.getElementById('genStatus');
+    const error = document.getElementById('genError');
+    const output = document.getElementById('draftOutput');
+
+    btn.disabled = true;
+    genText.textContent = 'Generating...';
+    status.classList.remove('hidden');
+    error.classList.add('hidden');
+    output.classList.add('hidden');
+
+    fetch(`/proposals/${proposalId}/generate`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+    })
+    .then(r => r.json())
+    .then(data => {
+        status.classList.add('hidden');
+        if (data.error) {
+            error.classList.remove('hidden');
+            document.getElementById('errorText').textContent = data.error;
+            btn.disabled = false;
+            genText.textContent = 'Retry';
+        } else {
+            generatedText = data.text;
+            generatedFilename = data.filename || 'generated-draft.txt';
+            document.getElementById('draftText').textContent = data.text;
+            output.classList.remove('hidden');
+            genText.textContent = 'Done';
+        }
+    })
+    .catch(err => {
+        status.classList.add('hidden');
+        error.classList.remove('hidden');
+        document.getElementById('errorText').textContent = 'Network error: ' + err.message;
+        btn.disabled = false;
+        genText.textContent = 'Retry';
+    });
+}
+
+function downloadDraft() {
+    if (!generatedText) return;
+    const blob = new Blob([generatedText], { type: 'text/plain' });
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement('a');
+    a.href = url;
+    a.download = generatedFilename;
+    document.body.appendChild(a);
+    a.click();
+    document.body.removeChild(a);
+    URL.revokeObjectURL(url);
+}
+</script>
 {% endblock %}