diff --git a/data/drafts.db b/data/drafts.db index 2485f1e..1c08533 100644 Binary files a/data/drafts.db and b/data/drafts.db differ diff --git a/scripts/pipeline-refresh.sh b/scripts/pipeline-refresh.sh new file mode 100755 index 0000000..8ff6a67 --- /dev/null +++ b/scripts/pipeline-refresh.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Pipeline refresh: extract missing ideas, re-score, then re-run gap analysis +# Run from project root: bash scripts/pipeline-refresh.sh + +set -euo pipefail +cd "$(dirname "$0")/.." +export PYTHONPATH=src + +echo "=== Step 1/3: Extract ideas from 38 remaining drafts ===" +python -m ietf_analyzer.cli ideas --all --cheap --batch 5 --limit 50 + +echo "" +echo "=== Step 2/3: Score new ideas for novelty ===" +python -m ietf_analyzer.cli ideas score --cheap --batch 10 + +echo "" +echo "=== Step 3/3: Re-run gap analysis ===" +python -m ietf_analyzer.cli gaps --refresh + +echo "" +echo "=== Done ===" +python -c " +from ietf_analyzer.config import Config +from ietf_analyzer.db import Database +cfg = Config.load() +db = Database(cfg) +ideas = db.conn.execute('SELECT count(*) FROM ideas').fetchone()[0] +gaps = db.conn.execute('SELECT count(*) FROM gaps').fetchone()[0] +print(f'Ideas: {ideas}, Gaps: {gaps}') +" diff --git a/src/webui/blueprints/admin.py b/src/webui/blueprints/admin.py index 25c2b5f..a3f75df 100644 --- a/src/webui/blueprints/admin.py +++ b/src/webui/blueprints/admin.py @@ -119,6 +119,7 @@ def gap_detail(gap_id: int): generated = get_generated_drafts() gap_proposals = get_proposals_for_gap(db(), gap_id) related_drafts = get_drafts_for_gap(db(), gap_id) + return render_template("gap_detail.html", gap=gap, generated_drafts=generated, proposals=gap_proposals, related_drafts=related_drafts) @@ -514,6 +515,53 @@ def proposal_delete(proposal_id): return redirect(url_for("admin.proposals")) +@admin_bp.route("/proposals//generate", methods=["POST"]) +@admin_required +def proposal_generate(proposal_id): + """Generate an Internet-Draft from a proposal and its linked gaps.""" + proposal = get_proposal_detail(db(), proposal_id) + if not proposal: + return jsonify({"error": "Proposal not found"}), 404 + + try: + from ietf_analyzer.config import Config + from ietf_analyzer.analyzer import Analyzer + from ietf_analyzer.draftgen import DraftGenerator + + cfg = Config.load() + database = db() + analyzer = Analyzer(cfg, database) + generator = DraftGenerator(cfg, database, analyzer) + + # Build rich topic from proposal + linked gaps + topic = proposal["title"] + gap_context = "" + if proposal.get("gaps"): + gap_topics = [g["topic"] for g in proposal["gaps"]] + gap_context = "\n\nThis proposal addresses these gaps:\n" + "\n".join( + f"- {g['topic']}: {g['description'][:200]}" for g in proposal["gaps"] + ) + if proposal.get("description"): + gap_context += f"\n\nProposal description: {proposal['description']}" + + slug = proposal.get("slug", "proposal")[:40] + output_path = str( + Path(_project_root) / "data" / "reports" / "generated-drafts" + / f"draft-proposal-{proposal_id}-{slug}.txt" + ) + path = generator.generate(topic + gap_context, output_path=output_path) + draft_text = Path(path).read_text(errors="replace") + + return jsonify({ + "success": True, + "text": draft_text, + "filename": Path(path).name, + "path": path, + }) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + @admin_bp.route("/api/proposals") @admin_required def api_proposals(): diff --git a/src/webui/templates/about.html b/src/webui/templates/about.html index 8ef2440..5b6dc3b 100644 --- a/src/webui/templates/about.html +++ b/src/webui/templates/about.html @@ -10,14 +10,21 @@

What is this?

- A tool for tracking, categorizing, rating, and mapping IETF Internet-Drafts - focused on AI and agent-related topics. It uses Claude for analysis and rating, - Ollama for embeddings, and SQLite for storage. + A research tool for tracking, categorizing, rating, and mapping standardization + documents on AI and agent-related topics across six standards bodies: + IETF, + ISO/IEC, + ITU-T, + ETSI, + NIST, and + W3C. + It uses Claude for analysis and rating, Ollama for embeddings, and SQLite for storage.

- The dashboard provides interactive visualizations of the draft landscape, + The dashboard provides interactive visualizations of the standardization landscape, including category breakdowns, rating distributions, author networks, - extracted ideas, and gap analysis. + extracted ideas, and gap analysis — answering the question: + Where is the AI agent standards race heading, and what's missing?

@@ -54,10 +61,12 @@

Data Collection Methodology

- Drafts are discovered by searching the + IETF drafts are discovered via the IETF Datatracker API - for documents whose abstract contains any of the following keywords. - Only drafts submitted since {{ fetch_since }} are included. + by searching abstracts for the keywords below + (only drafts since {{ fetch_since }}). + ISO, ITU-T, ETSI, NIST, and W3C documents are sourced from their respective public catalogs + using related search terms.

Search Keywords

@@ -124,7 +133,7 @@
  • Embeddings: Ollama (nomic-embed-text)
  • Storage: SQLite with FTS5 full-text search
  • Dashboard: Flask, Tailwind CSS, Plotly.js
    • -
  • Data source: IETF Datatracker API
    • +
  • Data sources: IETF Datatracker, ISO, ITU-T, ETSI, NIST, W3C
    • diff --git a/src/webui/templates/gap_detail.html b/src/webui/templates/gap_detail.html index bca987d..4ec1ac9 100644 --- a/src/webui/templates/gap_detail.html +++ b/src/webui/templates/gap_detail.html @@ -3,27 +3,7 @@ {% block title %}{{ gap.topic }} — Gap Explorer{% endblock %} -{% block extra_head %} - -{% endblock %} +{% block extra_head %}{% endblock %} {% block content %} @@ -71,19 +51,14 @@ -
    - - - Search related drafts - - {% if gap.category %} - | + {% if gap.category %} +
    Browse {{ gap.category }} drafts - {% endif %}
    + {% endif %}
    @@ -169,126 +144,16 @@ {% endif %} - + +{% if generated_drafts %}
    -
    -
    -

    Generate Internet-Draft

    -

    Use AI to generate a full Internet-Draft addressing this gap

    -
    - -
    - - - - - - - - - - - -
    - Want to see what generated drafts look like without waiting? - View the demo page - with {{ generated_drafts | length }} pre-generated examples. -
    +

    Generated Drafts

    +

    + {{ generated_drafts | length }} pre-generated example{{ 's' if generated_drafts | length != 1 }}. + View all +

    +{% endif %} {% endblock %} -{% block extra_scripts %} - -{% endblock %} +{% block extra_scripts %}{% endblock %} diff --git a/src/webui/templates/overview.html b/src/webui/templates/overview.html index 386f3bb..abb3477 100644 --- a/src/webui/templates/overview.html +++ b/src/webui/templates/overview.html @@ -9,7 +9,7 @@

    Dashboard Overview

    -

    IETF AI/Agent Internet-Drafts at a glance. Drafts are fetched from the IETF Datatracker, then analyzed by Claude AI across five dimensions (novelty, maturity, overlap, momentum, relevance) to produce a composite score from 1.0 to 5.0.

    +

    AI/Agent standardization landscape at a glance. 761 documents from IETF, ISO, ITU-T, ETSI, NIST, and W3C — analyzed by Claude AI across five dimensions (novelty, maturity, overlap, momentum, relevance) to produce a composite score from 1.0 to 5.0.

    {% if is_admin %} {% if proposal.content_md %} -
    +

    Content

    {{ proposal.content_md }}
    {% endif %} + + +
    +
    +
    +

    Generate Internet-Draft

    +

    Use AI to generate a full IETF Internet-Draft from this proposal{% if proposal.gaps %} and its {{ proposal.gaps | length }} linked gap{{ 's' if proposal.gaps | length != 1 }}{% endif %}

    +
    + +
    + + + + + + + + + +
    +{% endblock %} + +{% block extra_scripts %} + {% endblock %}