From 757b781c67139eedfd44ff87cf73e9008131608f Mon Sep 17 00:00:00 2001 From: Christian Nennemann Date: Sat, 7 Mar 2026 20:52:56 +0100 Subject: [PATCH] Platform upgrade: semantic search, citations, readiness, tests, Docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Major features added by 5 parallel agent teams: - Semantic "Ask" (NL queries via FTS5 + embeddings + Claude synthesis) - Global search across drafts, ideas, authors, gaps - REST API expansion (14 endpoints, up from 3) with CSV/JSON export - Citation graph visualization (D3.js, 440 nodes, 2422 edges) - Standards readiness scoring (0-100 composite from 6 factors) - Side-by-side draft comparison view with shared/unique analysis - Annotation system (notes + tags per draft, DB-persisted) - Docker deployment (Dockerfile + docker-compose with Ollama) - Scheduled updates (cron script with log rotation) - Pipeline health dashboard (stage progress bars, cost tracking) - Test suite foundation (54 pytest tests covering DB, models, web data) Fixes: compare_drafts() stubbed→working, get_authors_for_draft() bug, source-aware analysis prompts, config env var overrides + validation, resilient batch error handling with --retry-failed, observatory --dry-run Co-Authored-By: Claude Opus 4.6 --- .dockerignore | 11 + Dockerfile | 24 ++ README.md | 32 ++ data/reports/dev-journal.md | 33 ++ data/reports/platform-improvement-plan.md | 182 ++++++++++ docker-compose.yml | 26 ++ pyproject.toml | 7 + scripts/scheduled-update.sh | 61 ++++ src/ietf_analyzer/analyzer.py | 101 +++++- src/ietf_analyzer/cli.py | 401 +++++++++++++++++++--- src/ietf_analyzer/config.py | 54 ++- src/ietf_analyzer/db.py | 105 +++++- src/ietf_analyzer/observatory.py | 74 +++- src/ietf_analyzer/readiness.py | 102 ++++++ src/ietf_analyzer/search.py | 280 +++++++++++++++ src/webui/app.py | 269 ++++++++++++++- src/webui/data.py | 380 +++++++++++++++++++- src/webui/templates/ask.html | 153 +++++++++ src/webui/templates/base.html | 19 + src/webui/templates/citations.html | 392 +++++++++++++++++++++ src/webui/templates/comparison.html | 220 ++++++++++++ src/webui/templates/draft_detail.html | 151 ++++++++ src/webui/templates/drafts.html | 138 +++++++- src/webui/templates/gaps.html | 6 +- src/webui/templates/idea_clusters.html | 177 +++++++--- src/webui/templates/ideas.html | 8 + src/webui/templates/monitor.html | 65 ++++ src/webui/templates/search_results.html | 149 ++++++++ tests/__init__.py | 0 tests/conftest.py | 168 +++++++++ tests/test_db.py | 287 ++++++++++++++++ tests/test_models.py | 190 ++++++++++ tests/test_web_data.py | 158 +++++++++ 33 files changed, 4253 insertions(+), 170 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 data/reports/platform-improvement-plan.md create mode 100644 docker-compose.yml create mode 100755 scripts/scheduled-update.sh create mode 100644 src/ietf_analyzer/readiness.py create mode 100644 src/ietf_analyzer/search.py create mode 100644 src/webui/templates/ask.html create mode 100644 src/webui/templates/citations.html create mode 100644 src/webui/templates/comparison.html create mode 100644 src/webui/templates/search_results.html create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_db.py create mode 100644 tests/test_models.py create mode 100644 tests/test_web_data.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..230d824 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +__pycache__ +*.pyc +.env +paper/ +.claude/ +*.egg-info +.mypy_cache +.pytest_cache +data/ietf_drafts.db +docs/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ff6d264 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +# Copy project metadata and source first for layer caching +COPY pyproject.toml . +COPY src/ src/ + +# Install the package and all dependencies (including flask) +RUN pip install --no-cache-dir . + +# Copy data directory (DB, config, reports) +COPY data/ data/ + +ENV PYTHONUNBUFFERED=1 + +EXPOSE 5000 + +CMD ["python", "src/webui/app.py"] diff --git a/README.md b/README.md index 10cb9c1..465fe67 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,34 @@ ietf viz all # Open the interactive browser xdg-open data/figures/browser.html + +# Launch the web dashboard +./scripts/run-webui.sh ``` +## Web Dashboard + +A full interactive dashboard at `http://127.0.0.1:5000` with 8 pages: + +```bash +# Start the dashboard +./scripts/run-webui.sh +# or: python src/webui/app.py +``` + +| Page | What it shows | +|------|---------------| +| **Overview** | Stat cards, score histogram, category donut, submission timeline, category radar | +| **Draft Explorer** | Searchable/filterable/sortable table of all drafts with category pills and score badges | +| **Draft Detail** | Individual draft view with score ring, dimension bars, ideas, references, and linked authors | +| **Ratings** | Score distributions, dimension box plots, category radar, novelty vs maturity scatter, top-20 leaderboard | +| **Landscape** | t-SNE embedding map, quality quadrants, violin plots by category | +| **Authors** | Co-authorship force-directed graph, organization charts, cross-org collaboration | +| **Ideas** | Extracted ideas grouped by type with search | +| **Gaps** | Gap cards sorted by severity with links to related drafts | + +Charts are interactive (Plotly.js) — click data points to navigate to draft details, click categories to filter. + ## CLI Commands ### Core Pipeline @@ -182,6 +208,7 @@ The safety deficit is the most striking finding — only **12.3%** of categorize - **SQLite** with FTS5 full-text search and WAL mode - **Anthropic Claude** (Sonnet 4) for analysis, rating, idea extraction, gap analysis - **Ollama** (nomic-embed-text) for local embeddings and similarity +- **Flask** with Jinja2 for the interactive web dashboard - **Plotly** for interactive HTML visualizations - **Matplotlib/Seaborn** for publication-ready static figures - **NetworkX** for author collaboration graph analysis @@ -203,6 +230,11 @@ src/ietf_analyzer/ draftgen.py # Internet-Draft generation from gap analysis config.py # Configuration with defaults +src/webui/ + app.py # Flask application with all routes + data.py # Data access layer (stats, filtering, t-SNE, network graphs) + templates/ # Jinja2 templates (base + 8 page templates) + data/ drafts.db # SQLite database (all analysis data) reports/ # Generated markdown reports diff --git a/data/reports/dev-journal.md b/data/reports/dev-journal.md index c5522c7..5aa96ca 100644 --- a/data/reports/dev-journal.md +++ b/data/reports/dev-journal.md @@ -4,6 +4,19 @@ --- +### 2026-03-07 CODER C — Citation Graph, Readiness Scoring, Annotations, Data Surfacing + +**What**: Implemented four features in a single session: +1. **Citation Graph Visualization** (`/citations`): D3.js force-directed graph showing cross-references between drafts and RFCs. Nodes colored by type (blue=draft, orange=RFC), sized by influence (in-degree). Includes category filter, min-refs slider, hover tooltips, click-to-navigate, and a top-referenced RFCs table. New `get_citation_graph()` in data.py, route + API endpoint in app.py. +2. **Standards Readiness Scoring**: New `readiness.py` module computing a 0-100 composite score from 6 weighted factors (WG adoption 25%, revision maturity 15%, reference density 15%, cited-by count 15%, author experience 15%, momentum rating 15%). Displayed as a progress gauge on draft detail pages, added as sortable column on drafts listing, and shown in `ietf show` CLI output. +3. **Annotation System**: New `annotations` table in DB schema with `upsert_annotation`, `get_annotation`, `get_all_annotations`, `search_by_tag` methods. New `ietf annotate` CLI command with `--note`, `--tag`, `--remove-tag` options. Web UI: inline note editor + tag chips with add/remove on draft detail page, backed by POST `/api/drafts//annotate` endpoint. +4. **Surface Underutilized Data**: Exposed `novelty_score` (from pipeline/quality.py) in ideas.html and draft_detail.html as color-coded N:X badges. Gap severity now sorts critical-first (was alphabetical). `all_ideas()` and `get_ideas_for_draft()` now return `novelty_score` field. + +**Why**: These features leverage existing data (4231 refs, novelty scores, severity) that was computed but never surfaced to users. Readiness scoring gives a quick at-a-glance RFC proximity signal. Annotations enable user workflow. +**Result**: 8 files modified (db.py, data.py, app.py, cli.py, base.html, draft_detail.html, ideas.html, drafts.html, gaps.html), 2 files created (readiness.py, citations.html). Citations link added to sidebar nav. + +--- + ### 2026-03-06 CODER — Interactive D3.js Author Network Visualization **What**: Replaced the Plotly spring-layout co-authorship graph on `/authors` with a full D3.js v7 force-directed network. Added enriched data layer (`get_author_network_full`) with avg draft scores per author, connected-component cluster detection (68 clusters found), and a new `/api/authors/network` JSON endpoint. Template now includes: interactive D3 force graph with zoom/pan/drag, org filter dropdown, cluster highlighting with zoom-to-fit, hover tooltips showing author details + draft list, click-to-navigate, plus the existing Plotly org bar chart, cross-org collaboration chart, sortable authors table (now top 50), and org stats sidebar. @@ -411,3 +424,23 @@ - Decisions made: **GitHub Pages** for publication, **staggered 1/day** cadence, **MIT license** - Agent utilization: Architect (2 tasks, shut down), Writer (2 tasks, shut down), Planner (1 task, shut down) **Surprise**: The crash recovery was seamless — the dev journal served exactly its intended purpose. Every agent could read the journal and understand the full state without any human explanation. The journal-as-coordination-mechanism is the strongest vindication of the CLAUDE.md journaling requirement. This should feature prominently in Post 8. + +### 2026-03-07 CODER E — W3C Integration, Docker, Scheduling, Pipeline Health + +**What**: Four-part infrastructure sprint to make the platform multi-source, self-running, and deployable: + +1. **W3C Integration** — Wired the existing W3C fetcher (`sources/w3c.py`) into the full pipeline. Made analysis prompts source-aware (`_doc_type_label()` returns "IETF draft" or "W3C specification" based on `source` field). Added source filter dropdown (IETF / W3C / All) to the Draft Explorer web UI with colored source badges (blue for IETF, green for W3C). Updated `get_drafts_page()` to accept `source` parameter. All pagination and sort links preserve source filter state. Config documents how to enable W3C: `ietf observatory update --source w3c` or add `"w3c"` to `observatory_sources` in config.json. + +2. **Scheduled Updates** — Created `scripts/scheduled-update.sh` for cron-based automation. Handles .env loading, log rotation (30 days), and proper error exit codes. Usage: `crontab -e -> 0 6 * * * /path/to/scheduled-update.sh` + +3. **Docker Deployment** — Created `Dockerfile` (python:3.11-slim), `docker-compose.yml` (web + ollama services with volume mounts for data persistence), and `.dockerignore`. One-command deployment: `docker compose up`. + +4. **Pipeline Health** — Enhanced `ietf pipeline status` to show comprehensive health: processing stage breakdown (rated/embedded/ideas with ASCII progress bars), total ideas, gaps, API token usage, estimated cost. Enhanced monitor web page with visual pipeline progress bars, cost tracking panel, and document/idea/gap counts. Added `--dry-run` flag to `ietf observatory update` that previews what would happen. Wrapped all observatory update steps in try/except for graceful error recovery — failures in one stage no longer block subsequent stages. + +**Why**: The platform was IETF-only despite having a complete W3C fetcher. Docker makes deployment reproducible. Scheduled updates make it self-running. Error recovery prevents partial failures from wasting an entire update cycle. + +**Result**: +- Files modified: `analyzer.py`, `observatory.py`, `cli.py`, `config.py`, `data.py`, `app.py`, `drafts.html`, `monitor.html` +- Files created: `Dockerfile`, `docker-compose.yml`, `.dockerignore`, `scripts/scheduled-update.sh` +- All Python files compile cleanly +- No breaking changes to existing IETF-only workflows diff --git a/data/reports/platform-improvement-plan.md b/data/reports/platform-improvement-plan.md new file mode 100644 index 0000000..46b837d --- /dev/null +++ b/data/reports/platform-improvement-plan.md @@ -0,0 +1,182 @@ +# Platform Improvement Plan: IETF Draft Analyzer → Standards Intelligence Platform + +*Generated 2026-03-07 — Based on full codebase audit and architectural analysis* + +--- + +## Current State Summary + +| Dimension | What Exists | Assessment | +|-----------|-------------|------------| +| **Data** | 434 drafts, 557 authors, 419 ideas, 11 gaps, 4231 refs to 694 RFCs | Strong foundation | +| **CLI** | 20+ commands (fetch, analyze, embed, ideas, gaps, report, viz, wg, etc.) | Feature-rich | +| **Web UI** | 16 Flask pages with D3.js/Plotly visualizations | Good but disconnected | +| **Pipeline** | Observatory class with multi-source support | Built but manual | +| **Multi-SDO** | IETF complete, W3C fetcher written but unused | Partially built | +| **Tests** | Zero tests | Critical gap | +| **Deployment** | Manual Python setup | No containerization | + +## The Transformation + +**From**: A powerful CLI analysis tool that an expert runs manually +**To**: A living intelligence platform that monitors, alerts, and answers questions + +--- + +## Phase 1: Quick Wins (All Parallelizable) + +### 1.1 Global Search [S: 3-5h] +Add a unified search bar to the web UI that queries across drafts (FTS5), ideas, authors, and gaps simultaneously. Currently search is isolated to the drafts page. + +- **Files**: `src/webui/app.py` (new `/search` route), `src/webui/data.py` (new `global_search()`), `src/webui/templates/base.html` (search in sidebar), new `search_results.html` +- **Why first**: Most common user action on any data platform + +### 1.2 REST API [S: 2-3h] +Expose all existing `data.py` functions as JSON endpoints. Only 3 API endpoints exist today — extend to all 15+ data views. + +- **Files**: `src/webui/app.py` (add `/api/ideas`, `/api/gaps`, `/api/ratings`, `/api/timeline`, `/api/landscape`, `/api/similarity`, `/api/drafts/`, etc.) +- **Why**: Enables programmatic access, third-party tools, decouples data from presentation + +### 1.3 Export (CSV/JSON) [S: 3-4h] +Add export buttons to web UI pages + `ietf export` CLI command. + +- **Files**: `src/ietf_analyzer/cli.py` (new `export` command), API endpoints get `?format=csv` support +- **Depends on**: 1.2 + +### 1.4 Annotation System [M: 4-6h] +Add private notes and custom tags per draft, persisted in DB. + +- **Files**: `src/ietf_analyzer/db.py` (new `annotations` table), `src/webui/app.py` (POST endpoint), `src/webui/templates/draft_detail.html` (inline edit), `src/ietf_analyzer/cli.py` (new `annotate` command) +- **Why**: Analysts need to layer their own context onto the data + +### 1.5 Test Suite Foundation [M: 6-8h] +Create pytest infrastructure with ~30 tests covering DB layer, models, and web data functions. + +- **Files**: new `tests/conftest.py`, `tests/test_db.py`, `tests/test_models.py`, `tests/test_data.py`, update `pyproject.toml` +- **Why**: Zero tests today. Every future change benefits from a safety net + +--- + +## Phase 2: Core Platform + +### 2.1 Semantic Search / "Ask" [M: 1-2 days] — THE SIGNATURE FEATURE +Natural language queries: "Which drafts address agent authentication?" → synthesized answer with citations. + +- Embed the query via Ollama, compute cosine similarity against all 434 draft embeddings +- Merge with FTS5 keyword results using reciprocal rank fusion +- Optionally synthesize answer via Claude with top-K context +- **Files**: new `src/ietf_analyzer/search.py`, `src/ietf_analyzer/cli.py` (`ietf ask`), `src/webui/app.py` (`/ask` route), new template + +### 2.2 Competitive Landscape Mapping [M: 1-2 days] +Auto-detect and compare competing proposals in the same problem space. + +- Group drafts by similarity clusters, enrich with rating comparisons and WG adoption status +- Show head-to-head comparisons: where they agree, where they diverge +- **Files**: new `src/ietf_analyzer/competition.py`, `src/webui/app.py` (`/competition` route), new template + +### 2.3 Standards Readiness Scoring [M: 1 day] +Composite 0-100 "readiness" score: WG adoption, revision count, reference density, cited-by count, author track record. + +- **Files**: new `src/ietf_analyzer/readiness.py`, update `src/webui/templates/draft_detail.html` (gauge chart), update drafts listing + +### 2.4 Scheduled Updates + Pipeline Health [M: 1 day] +Cron-based auto-fetch using existing Observatory, plus monitoring dashboard. + +- **Files**: new `scripts/scheduled-update.sh`, enhance `src/webui/templates/monitor.html` (stage breakdown, cost tracking, failure log) + +### 2.5 Comparison View [M: 1 day] +Side-by-side comparison of 2+ drafts with rating radar overlay, shared/unique ideas, shared/unique references. + +- **Files**: `src/webui/app.py` (`/compare?drafts=...`), `src/webui/data.py` (`get_comparison_data()`), new template, add checkboxes to drafts listing + +--- + +## Phase 3: Intelligence + +### 3.1 Trend Forecasting [L: 2-3 days] +Predict which areas will grow based on submission velocity, revision activity, WG adoption signals. + +- Per-category momentum signals, linear/exponential extrapolation, "Hot/Cooling/Emerging" tags +- **Files**: new `src/ietf_analyzer/forecasting.py`, new web page `/trends` + +### 3.2 Change Detection & Diffing [L: 2-3 days] +Track what changed between draft revisions, summarize changes via Claude. + +- New `draft_revisions` table to archive old versions +- Section-level diff with Claude-generated change summaries +- **Files**: new `src/ietf_analyzer/diff.py`, update `src/ietf_analyzer/db.py`, `src/ietf_analyzer/fetcher.py` +- **Depends on**: 2.4 (scheduled updates to catch new revisions) + +### 3.3 Citation Graph [M: 1-2 days] +Visual dependency tree from the 4231 existing cross-references to 694 RFCs. + +- D3.js force-directed graph (pattern from authors.html), PageRank-style influence scores +- **Files**: `src/webui/data.py` (`get_citation_graph()`), new template `citations.html` + +### 3.4 Newsletter Generation [M: 1-2 days] +Automated weekly/monthly digest with new drafts, significant changes, trend shifts. + +- **Files**: new `src/ietf_analyzer/newsletter.py`, `src/ietf_analyzer/cli.py` (`ietf newsletter`) +- **Depends on**: 3.2 (for change content) + +--- + +## Phase 4: Scale + +### 4.1 Docker Deployment [S: 3-4h] +Dockerfile + docker-compose.yml with Flask app + Ollama. + +### 4.2 Complete W3C Integration [M: 1 day] +Wire existing W3C fetcher into Observatory pipeline, make prompts source-aware, add source filter to web UI. + +### 4.3 IEEE + 3GPP Sources [L: 3-5 days] +New source fetchers following `SourceFetcher` protocol. Depends on 4.2 validating the pipeline. + +### 4.4 Cross-SDO Analysis [L: 2-3 days] +Compare work across standards bodies. Embedding similarity between IETF/W3C/IEEE specs, gap analysis for topics only one body covers. + +### 4.5 Plugin Architecture [L: 2-3 days] +Formalize extension points for sources, analyzers, report types via Python entry points. + +--- + +## Recommended Agent Team Assignments + +### Immediate Sprint (Phase 1 — all in parallel) + +| Agent | Items | Focus | +|-------|-------|-------| +| **Coder A** | 1.1 + 1.2 | Global search + REST API | +| **Coder B** | 1.4 + 1.3 | Annotations + Export | +| **Coder C** | 1.5 | Test suite foundation | + +### Next Sprint (Phase 2 — mostly parallel) + +| Agent | Items | Focus | +|-------|-------|-------| +| **Coder A** | 2.1 | Semantic search / Ask (signature feature) | +| **Coder B** | 2.2 + 2.5 | Competition mapping + Comparison view | +| **Coder C** | 2.3 + 2.4 | Readiness scoring + Pipeline health | + +--- + +## Impact vs Effort Matrix + +``` +HIGH IMPACT + | + | 2.1 Ask 3.2 Diffing + | 1.1 Search 3.1 Forecasting + | 2.2 Competition 4.4 Cross-SDO + | 2.5 Compare + | 1.2 API + | 2.3 Readiness 3.3 Citations + | 1.4 Annotations 3.4 Newsletter + | 1.3 Export 4.2 W3C + | 1.5 Tests 4.1 Docker + | 4.5 Plugins + | 4.3 IEEE/3GPP +LOW IMPACT + +------------------------------------------ + LOW EFFORT HIGH EFFORT +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..cf24fc9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +version: '3.8' + +services: + web: + build: . + ports: + - "5000:5000" + volumes: + - ./data:/app/data + environment: + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - IETF_ANALYZER_OLLAMA_URL=http://ollama:11434 + depends_on: + - ollama + restart: unless-stopped + + ollama: + image: ollama/ollama + volumes: + - ollama_data:/root/.ollama + ports: + - "11434:11434" + restart: unless-stopped + +volumes: + ollama_data: diff --git a/pyproject.toml b/pyproject.toml index e6d4d50..4ce0da6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,10 +22,17 @@ dependencies = [ "scikit-learn>=1.3", "networkx>=3.2", "markdown>=3.5", + "flask>=3.0", ] +[project.optional-dependencies] +test = ["pytest", "pytest-cov"] + [project.scripts] ietf = "ietf_analyzer.cli:main" [tool.setuptools.packages.find] where = ["src"] + +[tool.pytest.ini_options] +pythonpath = ["src"] diff --git a/scripts/scheduled-update.sh b/scripts/scheduled-update.sh new file mode 100755 index 0000000..579576d --- /dev/null +++ b/scripts/scheduled-update.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Scheduled observatory update -- run via cron +# Usage: crontab -e -> 0 6 * * * /path/to/scheduled-update.sh +# +# This script runs the full observatory update cycle: +# 1. Fetch new docs from all enabled sources (IETF, W3C) +# 2. Analyze unrated documents with Claude +# 3. Generate embeddings with Ollama +# 4. Extract ideas from new documents +# 5. Re-run gap analysis if enough new docs +# +# Logs are saved to data/logs/update-YYYYMMDD-HHMMSS.log +set -euo pipefail + +cd "$(dirname "$0")/.." + +LOG_DIR="data/logs" +mkdir -p "$LOG_DIR" +LOG_FILE="$LOG_DIR/update-$(date +%Y%m%d-%H%M%S).log" + +echo "Starting scheduled update at $(date)" | tee "$LOG_FILE" + +# Load environment (API keys, etc.) +if [ -f .env ]; then + set -a + source .env + set +a +fi + +# Run the observatory update (delta mode -- only fetch new docs) +python -c " +import sys +sys.path.insert(0, 'src') +from ietf_analyzer.observatory import Observatory +from ietf_analyzer.analyzer import Analyzer +from ietf_analyzer.config import Config +from ietf_analyzer.db import Database + +config = Config.load() +db = Database(config) +analyzer = Analyzer(config, db) +obs = Observatory(config, db, analyzer) + +try: + result = obs.update() + print(f'Results: {result}') +except Exception as e: + print(f'ERROR: {e}', file=sys.stderr) + sys.exit(1) +finally: + db.close() +" >> "$LOG_FILE" 2>&1 + +EXIT_CODE=$? + +echo "Completed at $(date) (exit code: $EXIT_CODE)" | tee -a "$LOG_FILE" + +# Clean up old logs (keep last 30 days) +find "$LOG_DIR" -name "update-*.log" -mtime +30 -delete 2>/dev/null || true + +exit $EXIT_CODE diff --git a/src/ietf_analyzer/analyzer.py b/src/ietf_analyzer/analyzer.py index e7a9c90..3f80544 100644 --- a/src/ietf_analyzer/analyzer.py +++ b/src/ietf_analyzer/analyzer.py @@ -38,7 +38,7 @@ CATEGORIES_SHORT = [ # Compact prompt — abstract only, saves ~10x tokens vs full-text RATE_PROMPT_COMPACT = """\ -Rate this IETF draft. JSON only. +Rate this {doc_type}. JSON only. {name} | {title} | {time} | {pages}pg Abstract: {abstract} @@ -51,7 +51,7 @@ JSON only, no fences.""" # Batch prompt — rate multiple drafts in one call BATCH_PROMPT = """\ -Rate each IETF draft below. Return a JSON array with one object per draft, in order. +Rate each document below. Return a JSON array with one object per draft, in order. {drafts_block} @@ -62,14 +62,14 @@ Categories: {categories} Return ONLY a JSON array, no fences.""" COMPARE_PROMPT = """\ -Compare these IETF drafts — overlaps, unique ideas, complementary vs competing vs redundant. +Compare these documents — overlaps, unique ideas, complementary vs competing vs redundant. {drafts_section} Be specific about concrete mechanisms and design choices.""" EXTRACT_IDEAS_PROMPT = """\ -Extract discrete technical ideas and mechanisms from this IETF draft. +Extract discrete technical ideas and mechanisms from this {doc_type}. Return a JSON array. Each element: {{"title":"short name","description":"1-2 sentences","type":"mechanism|protocol|pattern|requirement|architecture|extension"}} {name} | {title} | {pages}pg @@ -81,7 +81,7 @@ Return 1-4 ideas. Extract only TOP-LEVEL novel contributions. Do NOT list sub-fe JSON array only, no fences.""" BATCH_IDEAS_PROMPT = """\ -Extract ideas from each IETF draft below. Return a JSON object mapping draft name -> array of ideas. +Extract ideas from each document below. Return a JSON object mapping document name -> array of ideas. Per idea: {{"title":"short name","description":"1 sentence","type":"mechanism|protocol|pattern|requirement|architecture|extension"}} {drafts_block} @@ -135,6 +135,15 @@ def _prompt_hash(text: str) -> str: return hashlib.sha256(text.encode()).hexdigest()[:16] +def _doc_type_label(source: str) -> str: + """Return a human-readable document type based on source.""" + labels = { + "ietf": "IETF draft", + "w3c": "W3C specification", + } + return labels.get(source, f"{source} document") + + class Analyzer: def __init__(self, config: Config | None = None, db: Database | None = None): self.config = config or Config.load() @@ -199,6 +208,7 @@ class Analyzer: return None prompt = RATE_PROMPT_COMPACT.format( + doc_type=_doc_type_label(draft.source), name=draft.name, title=draft.title, time=draft.date, pages=draft.pages or "?", abstract=draft.abstract[:2000], @@ -302,6 +312,7 @@ class Analyzer: console.print(f"Rating [bold]{len(unrated)}[/] drafts in batches of {batch_size}...") count = 0 + failures: list[tuple[str, str]] = [] with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), @@ -314,15 +325,29 @@ class Analyzer: batch = unrated[i:i + batch_size] names = ", ".join(d.name.split("-")[-1][:12] for d in batch) progress.update(task, description=f"Batch: {names}") - n = self.rate_batch(batch, batch_size=batch_size) - count += n + try: + n = self.rate_batch(batch, batch_size=batch_size) + count += n + except Exception as e: + batch_names = [d.name for d in batch] + for bn in batch_names: + failures.append((bn, str(e))) + console.print(f"[red]Batch failed: {e}[/]") progress.advance(task, advance=len(batch)) in_tok, out_tok = self.db.total_tokens_used() + total_attempted = len(unrated) console.print( f"Rated [bold green]{count}[/] drafts " f"| Total tokens used: {in_tok:,} in + {out_tok:,} out" ) + if failures: + console.print( + f"[yellow]Processed {count}/{total_attempted} drafts, " + f"{len(failures)} failure(s):[/]" + ) + for name, err in failures[:20]: + console.print(f" [red]{name}[/]: {err}") return count def extract_ideas(self, draft_name: str, use_cache: bool = True) -> list[dict] | None: @@ -337,6 +362,7 @@ class Analyzer: text_excerpt = draft.full_text[:3000] prompt = EXTRACT_IDEAS_PROMPT.format( + doc_type=_doc_type_label(draft.source), name=draft.name, title=draft.title, pages=draft.pages or "?", abstract=draft.abstract[:2000], @@ -451,6 +477,7 @@ class Analyzer: console.print(f"Extracting ideas from [bold]{len(missing)}[/] drafts ({model_label})...") count = 0 + failures: list[tuple[str, str]] = [] with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), @@ -465,23 +492,40 @@ class Analyzer: batch = missing[i:i + batch_size] names = ", ".join(n.split("-")[-1][:10] for n in batch) progress.update(task, description=f"Batch: {names}") - n = self.extract_ideas_batch(batch, cheap=cheap) - count += n + try: + n = self.extract_ideas_batch(batch, cheap=cheap) + count += n + except Exception as e: + for bn in batch: + failures.append((bn, str(e))) + console.print(f"[red]Batch failed: {e}[/]") progress.advance(task, advance=len(batch)) else: for name in missing: progress.update(task, description=f"Ideas: {name.split('-')[-1][:15]}") - result = self.extract_ideas(name) - if result: - count += 1 + try: + result = self.extract_ideas(name) + if result: + count += 1 + except Exception as e: + failures.append((name, str(e))) + console.print(f"[red]Failed {name}: {e}[/]") progress.advance(task) + total_attempted = len(missing) in_tok, out_tok = self.db.total_tokens_used() console.print( f"Extracted ideas from [bold green]{count}[/] drafts " f"({self.db.idea_count()} total ideas) " f"| Tokens: {in_tok:,} in + {out_tok:,} out" ) + if failures: + console.print( + f"[yellow]Processed {count}/{total_attempted} drafts, " + f"{len(failures)} failure(s):[/]" + ) + for name, err in failures[:20]: + console.print(f" [red]{name}[/]: {err}") return count def gap_analysis(self) -> list[dict]: @@ -551,28 +595,49 @@ class Analyzer: console.print(f"[red]Gap analysis failed: {e}[/]") return [] - def compare_drafts(self, draft_names: list[str]) -> str: - """Compare multiple drafts and return analysis text.""" + def compare_drafts(self, draft_names: list[str], use_cache: bool = True) -> dict: + """Compare multiple drafts and return structured comparison. + + Returns dict with keys: text, drafts (list of names that were compared), + or a dict with key 'error' on failure. + """ + valid_names = [] parts = [] for name in draft_names: draft = self.db.get_draft(name) if draft is None: console.print(f"[yellow]Skipping unknown draft: {name}[/]") continue + valid_names.append(name) parts.append(f"### {draft.title}\n**{name}**\n{draft.abstract}") if len(parts) < 2: - return "Need at least 2 valid drafts to compare." + return {"error": "Need at least 2 valid drafts to compare.", "drafts": valid_names} prompt = COMPARE_PROMPT.format( drafts_section="\n\n---\n\n".join(parts) ) + phash = _prompt_hash(prompt) + cache_key = "_compare_" + "_".join(sorted(valid_names)) + + # Check cache + if use_cache: + cached = self.db.get_cached_response(cache_key, phash) + if cached: + return {"text": cached, "drafts": valid_names} try: - text, _, _ = self._call_claude(prompt, max_tokens=2048) - return text + text, in_tok, out_tok = self._call_claude(prompt, max_tokens=2048) + + # Cache the result + self.db.cache_response( + cache_key, phash, self.config.claude_model, + prompt, text, in_tok, out_tok, + ) + + return {"text": text, "drafts": valid_names} except anthropic.APIError as e: - return f"Error: {e}" + return {"error": f"API error: {e}", "drafts": valid_names} def dedup_ideas(self, threshold: float = 0.85, dry_run: bool = True, draft_name: str | None = None) -> dict: diff --git a/src/ietf_analyzer/cli.py b/src/ietf_analyzer/cli.py index ccf3361..4f233f4 100644 --- a/src/ietf_analyzer/cli.py +++ b/src/ietf_analyzer/cli.py @@ -173,6 +173,20 @@ def show(name: str): else: console.print("[dim]Not yet rated — run: ietf analyze {name}[/]") + # Readiness score + from .readiness import compute_readiness + readiness = compute_readiness(db, name) + if readiness["score"] > 0: + console.print(f"\n[bold]Standards Readiness: [cyan]{readiness['score']}/100[/][/]") + rtable = Table(show_header=True) + rtable.add_column("Factor", width=20) + rtable.add_column("Value", justify="center", width=10) + rtable.add_column("Points", justify="right", width=8) + rtable.add_column("Detail") + for key, f in readiness["factors"].items(): + rtable.add_row(f["label"], f"{f['value']:.2f}", f"+{f['contribution']}", f["detail"]) + console.print(rtable) + # Save detailed report too path = reporter.draft_detail(name) if path: @@ -181,6 +195,56 @@ def show(name: str): db.close() +# ── annotate ───────────────────────────────────────────────────────────────── + + +@main.command() +@click.argument("draft_name") +@click.option("--note", "-n", default=None, help="Set/update the note text") +@click.option("--tag", "-t", multiple=True, help="Add a tag (can be used multiple times)") +@click.option("--remove-tag", "-r", multiple=True, help="Remove a tag (can be used multiple times)") +def annotate(draft_name: str, note: str | None, tag: tuple[str, ...], remove_tag: tuple[str, ...]): + """Add or view annotations (notes & tags) for a draft.""" + cfg = _get_config() + db = Database(cfg) + try: + draft = db.get_draft(draft_name) + if draft is None: + console.print(f"[red]Draft not found: {draft_name}[/]") + return + + # If no options, display current annotation + if note is None and not tag and not remove_tag: + ann = db.get_annotation(draft_name) + if ann: + console.print(f"\n[bold]Annotation for {draft_name}[/]") + console.print(f" Note: {ann['note'] or '(empty)'}") + console.print(f" Tags: {', '.join(ann['tags']) if ann['tags'] else '(none)'}") + console.print(f" Updated: {ann['updated_at']}") + else: + console.print(f"[dim]No annotation for {draft_name}. Use --note or --tag to add one.[/]") + return + + # Fetch existing tags for add/remove operations + existing = db.get_annotation(draft_name) + current_tags = existing["tags"] if existing else [] + + for t in tag: + if t not in current_tags: + current_tags.append(t) + for t in remove_tag: + if t in current_tags: + current_tags.remove(t) + + db.upsert_annotation(draft_name, note=note, tags=current_tags) + ann = db.get_annotation(draft_name) + console.print(f"[green]Annotation updated for {draft_name}[/]") + console.print(f" Note: {ann['note'] or '(empty)'}") + console.print(f" Tags: {', '.join(ann['tags']) if ann['tags'] else '(none)'}") + finally: + db.close() + + # ── analyze ────────────────────────────────────────────────────────────────── @@ -188,7 +252,8 @@ def show(name: str): @click.argument("name", required=False) @click.option("--all", "analyze_all", is_flag=True, help="Analyze all unrated drafts") @click.option("--limit", "-n", default=50, help="Max drafts to analyze (with --all)") -def analyze(name: str | None, analyze_all: bool, limit: int): +@click.option("--retry-failed", is_flag=True, help="Re-analyze drafts that previously failed (clears cache)") +def analyze(name: str | None, analyze_all: bool, limit: int, retry_failed: bool): """Analyze and rate drafts using Claude.""" from .analyzer import Analyzer @@ -197,7 +262,29 @@ def analyze(name: str | None, analyze_all: bool, limit: int): analyzer = Analyzer(cfg, db) try: - if analyze_all: + if retry_failed: + # Find drafts that have cache entries but no ratings (failed analyses) + unrated = db.unrated_drafts(limit=limit) + retryable = [] + for draft in unrated: + # Check if there's a cache entry for this draft (it was attempted) + row = db.conn.execute( + "SELECT COUNT(*) FROM llm_cache WHERE draft_name = ?", + (draft.name,), + ).fetchone() + if row[0] > 0: + retryable.append(draft) + if not retryable: + console.print("No previously failed drafts to retry.") + else: + console.print(f"Retrying [bold]{len(retryable)}[/] previously failed drafts...") + count = 0 + for draft in retryable: + rating = analyzer.rate_draft(draft.name, use_cache=False) + if rating: + count += 1 + console.print(f"Successfully re-analyzed [bold green]{count}[/] of {len(retryable)} drafts") + elif analyze_all: count = analyzer.rate_all_unrated(limit=limit) console.print(f"Analyzed [bold green]{count}[/] drafts") elif name: @@ -217,6 +304,62 @@ def analyze(name: str | None, analyze_all: bool, limit: int): db.close() +# ── ask ────────────────────────────────────────────────────────────────────── + + +@main.command() +@click.argument("question") +@click.option("--top", "-n", default=5, help="Number of source drafts to use") +@click.option("--cheap/--quality", default=True, help="Use Haiku (cheap) vs Sonnet (quality)") +def ask(question: str, top: int, cheap: bool): + """Ask a natural language question about the drafts. + + Examples: + ietf ask "Which drafts address agent authentication?" + ietf ask "What are the competing approaches to agent delegation?" --top 10 + ietf ask "How do safety mechanisms work?" --cheap + """ + from .search import HybridSearch + + cfg = _get_config() + db = Database(cfg) + + try: + searcher = HybridSearch(cfg, db) + console.print(f"\n[dim]Searching for relevant drafts...[/]") + result = searcher.ask(question, top_k=top, cheap=cheap) + + # Display the answer + console.print() + console.print("[bold cyan]Answer[/]") + console.print("[dim]" + "-" * 60 + "[/]") + console.print(result["answer"]) + console.print() + + # Display source drafts table + if result["sources"]: + table = Table(title="Source Drafts") + table.add_column("#", style="dim", width=3) + table.add_column("Draft", style="cyan", max_width=50) + table.add_column("Title", max_width=45) + table.add_column("Match", width=10) + table.add_column("Score", justify="right", width=8) + + for i, src in enumerate(result["sources"], 1): + score_str = f"{src['similarity']:.3f}" if src.get("similarity") else "-" + table.add_row( + str(i), + src["name"], + src["title"][:45], + src.get("match_type", ""), + score_str, + ) + + console.print(table) + finally: + db.close() + + # ── compare ────────────────────────────────────────────────────────────────── @@ -232,7 +375,12 @@ def compare(names: tuple[str, ...]): try: result = analyzer.compare_drafts(list(names)) - console.print(result) + if "error" in result: + console.print(f"[red]{result['error']}[/]") + else: + console.print(f"\n[bold cyan]Comparison of {len(result['drafts'])} drafts[/]") + console.print("[dim]" + "-" * 60 + "[/]") + console.print(result["text"]) finally: db.close() @@ -2107,7 +2255,8 @@ def draft_gen(gap_topic: str, output: str | None): @main.command("config") @click.option("--set", "set_key", nargs=2, help="Set a config key (e.g. --set claude_model claude-opus-4-20250514)") -def config_cmd(set_key: tuple[str, str] | None): +@click.option("--show", is_flag=True, help="Show effective config with env var sources noted") +def config_cmd(set_key: tuple[str, str] | None, show: bool): """Show or modify configuration.""" from dataclasses import asdict cfg = _get_config() @@ -2131,8 +2280,20 @@ def config_cmd(set_key: tuple[str, str] | None): console.print(f"[red]Unknown config key: {key}[/]") else: from dataclasses import asdict + env_sources = cfg.env_sources() for key, val in asdict(cfg).items(): - console.print(f" [bold]{key}:[/] {val}") + source_note = "" + if key in env_sources: + source_note = f" [yellow](from ${env_sources[key]})[/]" + console.print(f" [bold]{key}:[/] {val}{source_note}") + if env_sources: + console.print(f"\n [dim]({len(env_sources)} value(s) overridden by environment variables)[/]") + # Note about ANTHROPIC_API_KEY + import os + if os.environ.get("ANTHROPIC_API_KEY"): + console.print(" [dim]ANTHROPIC_API_KEY is set in environment[/]") + else: + console.print(" [dim]ANTHROPIC_API_KEY is NOT set in environment[/]") # ── pipeline ──────────────────────────────────────────────────────────────── @@ -2321,35 +2482,79 @@ def pipeline_quality(draft_id: int): @pipeline.command("status") def pipeline_status(): - """Show all generated drafts.""" + """Show pipeline health: processing stages, generated drafts, and API cost.""" cfg = _get_config() db = Database(cfg) try: - drafts = db.get_generated_drafts() - if not drafts: - console.print("No generated drafts yet. Run `ietf pipeline generate `") - return + # Pipeline health overview + total = db.count_drafts() + rated_count = len(db.drafts_with_ratings(limit=10000)) + unrated = len(db.unrated_drafts(limit=10000)) + unembedded = len(db.drafts_without_embeddings(limit=10000)) + embedded_count = total - unembedded + no_ideas = len(db.drafts_without_ideas(limit=10000)) + ideas_count = total - no_ideas + idea_total = db.idea_count() + gap_count = len(db.all_gaps()) + input_tok, output_tok = db.total_tokens_used() + est_cost = (input_tok * 3.0 / 1_000_000) + (output_tok * 15.0 / 1_000_000) - table = Table(title=f"Generated Drafts ({len(drafts)})") - table.add_column("ID", justify="right", width=4) - table.add_column("Draft Name", style="cyan") - table.add_column("Gap Topic") - table.add_column("Family", width=15) - table.add_column("Status", width=10) - table.add_column("Quality", justify="right", width=7) - table.add_column("Created", width=10) + # Last update + snapshots = db.get_snapshots(limit=1) + last_update = snapshots[0]["snapshot_at"][:19] if snapshots else "never" - for d in drafts: - table.add_row( - str(d["id"]), - d["draft_name"], - d["gap_topic"][:30], - d.get("family_name", ""), - d.get("status", "?"), - f"{d.get('quality_score', 0):.1f}" if d.get("quality_score") else "-", - (d.get("created_at") or "")[:10], - ) - console.print(table) + console.print("\n[bold]Pipeline Status[/]\n") + console.print(f" Total documents: [bold]{total}[/]") + console.print(f" Last update: {last_update}") + console.print() + + # Stage table + stage_table = Table(title="Processing Stages") + stage_table.add_column("Stage", width=20) + stage_table.add_column("Done", justify="right", width=8) + stage_table.add_column("Missing", justify="right", width=8) + stage_table.add_column("Progress", width=20) + + def bar(done, total_n): + pct = int(done / total_n * 100) if total_n > 0 else 0 + filled = pct // 5 + return f"[green]{'#' * filled}[/][dim]{'.' * (20 - filled)}[/] {pct}%" + + stage_table.add_row("Rated", str(rated_count), str(unrated), bar(rated_count, total)) + stage_table.add_row("Embedded", str(embedded_count), str(unembedded), bar(embedded_count, total)) + stage_table.add_row("Ideas extracted", str(ideas_count), str(no_ideas), bar(ideas_count, total)) + + console.print(stage_table) + + console.print(f"\n Total ideas: [bold]{idea_total}[/]") + console.print(f" Gaps identified: [bold]{gap_count}[/]") + console.print(f"\n API tokens: {input_tok:,} in + {output_tok:,} out") + console.print(f" Estimated cost: [bold]${est_cost:.2f}[/]") + + # Generated drafts + gen_drafts = db.get_generated_drafts() + if gen_drafts: + console.print() + table = Table(title=f"Generated Drafts ({len(gen_drafts)})") + table.add_column("ID", justify="right", width=4) + table.add_column("Draft Name", style="cyan") + table.add_column("Gap Topic") + table.add_column("Family", width=15) + table.add_column("Status", width=10) + table.add_column("Quality", justify="right", width=7) + table.add_column("Created", width=10) + + for d in gen_drafts: + table.add_row( + str(d["id"]), + d["draft_name"], + d["gap_topic"][:30], + d.get("family_name", ""), + d.get("status", "?"), + f"{d.get('quality_score', 0):.1f}" if d.get("quality_score") else "-", + (d.get("created_at") or "")[:10], + ) + console.print(table) finally: db.close() @@ -2397,28 +2602,38 @@ def observatory(): @observatory.command("update") @click.option("--source", "-s", default=None, help="Comma-separated sources (e.g. ietf,w3c)") @click.option("--full/--delta", default=False, help="Full refresh or delta only") -def observatory_update(source: str | None, full: bool): +@click.option("--dry-run", is_flag=True, default=False, help="Show what would happen without making changes") +def observatory_update(source: str | None, full: bool, dry_run: bool): """Fetch, analyze, and update the observatory.""" from .observatory import Observatory - from .analyzer import Analyzer cfg = _get_config() db = Database(cfg) - analyzer = Analyzer(cfg, db) try: - obs = Observatory(cfg, db, analyzer) + if dry_run: + obs = Observatory(cfg, db) + else: + from .analyzer import Analyzer + analyzer = Analyzer(cfg, db) + obs = Observatory(cfg, db, analyzer) sources = source.split(",") if source else None - console.print(f"[bold]Observatory update[/] ({'full' if full else 'delta'})") - result = obs.update(sources=sources, full=full) + mode = "full" if full else "delta" + console.print(f"[bold]Observatory update[/] ({mode}{' [DRY RUN]' if dry_run else ''})") + result = obs.update(sources=sources, full=full, dry_run=dry_run) - console.print(f"\n[bold green]Update complete![/]") - console.print(f" New docs: {result.get('new_docs', 0)}") - console.print(f" Analyzed: {result.get('analyzed', 0)}") - console.print(f" Embedded: {result.get('embedded', 0)}") - console.print(f" Ideas extracted: {result.get('ideas', 0)}") - if result.get("gaps_updated"): - console.print(f" Gaps re-analyzed: yes ({result.get('gap_count', 0)} gaps)") + if not dry_run: + console.print(f"\n[bold green]Update complete![/]") + console.print(f" New docs: {result.get('new_docs', 0)}") + console.print(f" Analyzed: {result.get('analyzed', 0)}") + console.print(f" Embedded: {result.get('embedded', 0)}") + console.print(f" Ideas extracted: {result.get('ideas', 0)}") + if result.get("gaps_changed"): + console.print(f" Gaps re-analyzed: yes") + if result.get("errors"): + console.print(f"\n [yellow]Errors ({len(result['errors'])}):[/]") + for err in result["errors"]: + console.print(f" - {err}") finally: db.close() @@ -2676,3 +2891,105 @@ def monitor_status(): console.print(table) finally: db.close() + + +# ── export ────────────────────────────────────────────────────────────────── + + +@main.command() +@click.option("--type", "export_type", type=click.Choice(["drafts", "ideas", "gaps", "authors", "ratings"]), + required=True, help="Type of data to export") +@click.option("--format", "fmt", type=click.Choice(["json", "csv"]), default="json", help="Output format") +@click.option("--output", "-o", "output_file", type=click.Path(), default=None, + help="Output file (default: stdout)") +def export(export_type: str, fmt: str, output_file: str | None): + """Export data as JSON or CSV.""" + import csv as csv_mod + import io + import json + + cfg = _get_config() + db = Database(cfg) + + try: + rows: list[dict] = [] + + if export_type == "drafts": + drafts = db.list_drafts(limit=10000, order_by="name ASC") + for d in drafts: + rating = db.get_rating(d.name) + row = { + "name": d.name, + "title": d.title, + "rev": d.rev, + "date": d.date, + "pages": d.pages or 0, + "group": d.group or "", + } + if rating: + row["score"] = round(rating.composite_score, 2) + row["novelty"] = rating.novelty + row["maturity"] = rating.maturity + row["overlap"] = rating.overlap + row["momentum"] = rating.momentum + row["relevance"] = rating.relevance + row["categories"] = json.dumps(rating.categories) + row["summary"] = rating.summary + rows.append(row) + + elif export_type == "ideas": + ideas = db.all_ideas() + rows = ideas + + elif export_type == "gaps": + gaps = db.all_gaps() + rows = gaps + + elif export_type == "authors": + top = db.top_authors(limit=10000) + for name, aff, cnt, drafts_list in top: + rows.append({ + "name": name, + "affiliation": aff, + "draft_count": cnt, + "drafts": json.dumps(drafts_list), + }) + + elif export_type == "ratings": + pairs = db.drafts_with_ratings(limit=10000) + for draft, rating in pairs: + rows.append({ + "name": draft.name, + "title": draft.title, + "score": round(rating.composite_score, 2), + "novelty": rating.novelty, + "maturity": rating.maturity, + "overlap": rating.overlap, + "momentum": rating.momentum, + "relevance": rating.relevance, + "categories": json.dumps(rating.categories), + "summary": rating.summary, + }) + + if fmt == "json": + text = json.dumps(rows, indent=2, ensure_ascii=False) + else: + # CSV + if not rows: + text = "" + else: + si = io.StringIO() + writer = csv_mod.DictWriter(si, fieldnames=rows[0].keys()) + writer.writeheader() + for row in rows: + writer.writerow(row) + text = si.getvalue() + + if output_file: + Path(output_file).write_text(text, encoding="utf-8") + console.print(f"Exported [bold green]{len(rows)}[/] {export_type} to [cyan]{output_file}[/] ({fmt})") + else: + click.echo(text) + + finally: + db.close() diff --git a/src/ietf_analyzer/config.py b/src/ietf_analyzer/config.py index 1882d07..6c9c958 100644 --- a/src/ietf_analyzer/config.py +++ b/src/ietf_analyzer/config.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os from dataclasses import dataclass, field, asdict from pathlib import Path @@ -24,6 +25,13 @@ DEFAULT_KEYWORDS = [ "aipref", ] +# Environment variable overrides (env var name -> config field name) +_ENV_OVERRIDES = { + "IETF_ANALYZER_DB_PATH": "db_path", + "IETF_ANALYZER_CLAUDE_MODEL": "claude_model", + "IETF_ANALYZER_OLLAMA_URL": "ollama_url", +} + @dataclass class Config: @@ -41,7 +49,9 @@ class Config: # Pipeline generation_max_tokens: int = 4096 generation_model: str = "" # defaults to claude_model - # Observatory + # Observatory — add "w3c" to enable W3C spec tracking: + # ietf observatory update --source w3c (one-off) + # or set observatory_sources to ["ietf", "w3c"] in config.json observatory_sources: list[str] = field(default_factory=lambda: ["ietf"]) dashboard_dir: str = str(DEFAULT_DATA_DIR.parent / "docs") w3c_groups: list[str] = field(default_factory=lambda: [ @@ -52,9 +62,47 @@ class Config: Path(self.data_dir).mkdir(parents=True, exist_ok=True) CONFIG_FILE.write_text(json.dumps(asdict(self), indent=2)) + def env_sources(self) -> dict[str, str]: + """Return {field_name: env_var_name} for fields overridden by env vars.""" + sources: dict[str, str] = {} + for env_var, field_name in _ENV_OVERRIDES.items(): + if os.environ.get(env_var): + sources[field_name] = env_var + return sources + + @classmethod + def _validate(cls, cfg: Config) -> None: + """Validate config values, raise ValueError on problems.""" + if not cfg.claude_model or not cfg.claude_model.strip(): + raise ValueError( + "claude_model must be a non-empty string. " + "Check your config file or IETF_ANALYZER_CLAUDE_MODEL env var." + ) + if not cfg.ollama_url.startswith(("http://", "https://")): + raise ValueError( + f"ollama_url must be an HTTP(S) URL, got: '{cfg.ollama_url}'. " + "Check your config file or IETF_ANALYZER_OLLAMA_URL env var." + ) + db_parent = Path(cfg.db_path).parent + if not db_parent.exists(): + raise ValueError( + f"db_path parent directory does not exist: '{db_parent}'. " + "Check your config file or IETF_ANALYZER_DB_PATH env var." + ) + @classmethod def load(cls) -> Config: if CONFIG_FILE.exists(): data = json.loads(CONFIG_FILE.read_text()) - return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__}) - return cls() + cfg = cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__}) + else: + cfg = cls() + + # Apply environment variable overrides (env vars take precedence) + for env_var, field_name in _ENV_OVERRIDES.items(): + env_val = os.environ.get(env_var) + if env_val is not None: + setattr(cfg, field_name, env_val) + + cls._validate(cfg) + return cfg diff --git a/src/ietf_analyzer/db.py b/src/ietf_analyzer/db.py index 9237092..f767602 100644 --- a/src/ietf_analyzer/db.py +++ b/src/ietf_analyzer/db.py @@ -192,6 +192,17 @@ CREATE TABLE IF NOT EXISTS gap_history ( recorded_at TEXT ); +-- Annotations (user notes + tags per draft) +CREATE TABLE IF NOT EXISTS annotations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + draft_name TEXT NOT NULL REFERENCES drafts(name), + note TEXT DEFAULT '', + tags TEXT DEFAULT '[]', + created_at TEXT, + updated_at TEXT, + UNIQUE(draft_name) +); + -- Monitor runs CREATE TABLE IF NOT EXISTS monitor_runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -529,14 +540,17 @@ class Database: ORDER BY da.author_order""", (draft_name,), ).fetchall() - cols = rows[0].keys() if rows else [] - return [Author( - person_id=r["person_id"], name=r["name"], - ascii_name=r["ascii_name"] if "ascii_name" in cols else "", - affiliation=r["affiliation"] if "affiliation" in cols else "", - resource_uri=r["resource_uri"] if "resource_uri" in cols else "", - fetched_at=r["fetched_at"] if "fetched_at" in cols else None, - ) for r in rows] + results = [] + for r in rows: + d = dict(r) + results.append(Author( + person_id=d["person_id"], name=d["name"], + ascii_name=d.get("ascii_name", ""), + affiliation=d.get("affiliation", ""), + resource_uri=d.get("resource_uri", ""), + fetched_at=d.get("fetched_at"), + )) + return results def drafts_without_authors(self, limit: int = 500) -> list[str]: rows = self.conn.execute( @@ -681,7 +695,8 @@ class Database: "SELECT * FROM ideas WHERE draft_name = ?", (draft_name,) ).fetchall() return [{"id": r["id"], "title": r["title"], "description": r["description"], - "type": r["idea_type"], "draft_name": r["draft_name"]} for r in rows] + "type": r["idea_type"], "draft_name": r["draft_name"], + "novelty_score": r["novelty_score"]} for r in rows] def delete_idea(self, idea_id: int) -> None: """Delete a single idea and its embedding by ID.""" @@ -706,7 +721,8 @@ class Database: "SELECT * FROM ideas ORDER BY draft_name" ).fetchall() return [{"title": r["title"], "description": r["description"], - "type": r["idea_type"], "draft_name": r["draft_name"]} for r in rows] + "type": r["idea_type"], "draft_name": r["draft_name"], + "novelty_score": r["novelty_score"]} for r in rows] def idea_count(self) -> int: return self.conn.execute("SELECT COUNT(*) FROM ideas").fetchone()[0] @@ -1380,6 +1396,75 @@ class Database: ).fetchone() return dict(row) if row else None + # --- Annotations --- + + def upsert_annotation(self, draft_name: str, note: str | None = None, tags: list[str] | None = None) -> None: + """Insert or update an annotation for a draft.""" + now = datetime.now(timezone.utc).isoformat() + existing = self.conn.execute( + "SELECT id, note, tags FROM annotations WHERE draft_name = ?", + (draft_name,), + ).fetchone() + if existing: + current_note = note if note is not None else existing["note"] + current_tags = tags if tags is not None else json.loads(existing["tags"] or "[]") + self.conn.execute( + "UPDATE annotations SET note = ?, tags = ?, updated_at = ? WHERE draft_name = ?", + (current_note, json.dumps(current_tags), now, draft_name), + ) + else: + self.conn.execute( + """INSERT INTO annotations (draft_name, note, tags, created_at, updated_at) + VALUES (?, ?, ?, ?, ?)""", + (draft_name, note or "", json.dumps(tags or []), now, now), + ) + self.conn.commit() + + def get_annotation(self, draft_name: str) -> dict | None: + """Return annotation for a draft, or None.""" + row = self.conn.execute( + "SELECT * FROM annotations WHERE draft_name = ?", (draft_name,) + ).fetchone() + if not row: + return None + return { + "id": row["id"], + "draft_name": row["draft_name"], + "note": row["note"], + "tags": json.loads(row["tags"] or "[]"), + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + def get_all_annotations(self) -> list[dict]: + """Return all annotations.""" + rows = self.conn.execute( + "SELECT * FROM annotations ORDER BY updated_at DESC" + ).fetchall() + return [ + { + "id": r["id"], + "draft_name": r["draft_name"], + "note": r["note"], + "tags": json.loads(r["tags"] or "[]"), + "created_at": r["created_at"], + "updated_at": r["updated_at"], + } + for r in rows + ] + + def search_by_tag(self, tag: str) -> list[str]: + """Return draft names that have a specific tag in their annotation.""" + rows = self.conn.execute( + "SELECT draft_name, tags FROM annotations" + ).fetchall() + results = [] + for r in rows: + tags = json.loads(r["tags"] or "[]") + if tag in tags: + results.append(r["draft_name"]) + return results + # --- Helpers --- @staticmethod diff --git a/src/ietf_analyzer/observatory.py b/src/ietf_analyzer/observatory.py index cc0eee4..9f683b8 100644 --- a/src/ietf_analyzer/observatory.py +++ b/src/ietf_analyzer/observatory.py @@ -76,6 +76,7 @@ class Observatory: self, sources: list[str] | None = None, full: bool = False, + dry_run: bool = False, ) -> dict: """Full update cycle. @@ -87,10 +88,30 @@ class Observatory: 6. Re-run gap analysis if >= 5 new docs 7. Record gap changes in gap_history 8. Return summary stats + + If dry_run=True, show what would be fetched/analyzed without doing it. """ sources = sources or self.config.observatory_sources stats: dict = {"sources": {}, "new_docs": 0, "analyzed": 0, "embedded": 0, "ideas": 0, "gaps_changed": False} + if dry_run: + console.print("[bold yellow]DRY RUN[/] — showing what would happen without making changes\n") + for src_name in sources: + console.print(f" Would fetch from: [cyan]{src_name}[/]") + src = self.db.get_source(src_name) + if src and src.get("last_fetch"): + console.print(f" Last fetch: {src['last_fetch'][:10]}") + else: + console.print(f" Last fetch: never (full fetch)") + unrated = len(self.db.unrated_drafts(limit=10000)) + unembedded = len(self.db.drafts_without_embeddings(limit=10000)) + no_ideas = len(self.db.drafts_without_ideas(limit=10000)) + console.print(f"\n Would analyze: [bold]{unrated}[/] unrated documents") + console.print(f" Would embed: [bold]{unembedded}[/] documents") + console.print(f" Would extract ideas from: [bold]{no_ideas}[/] documents") + stats["dry_run"] = True + return stats + # 1. Snapshot current state console.print("[bold]1/7[/] Creating snapshot...") snapshot_id = self.db.create_snapshot() @@ -99,35 +120,58 @@ class Observatory: console.print("[bold]2/7[/] Fetching from sources...") total_new = 0 for src_name in sources: - new_count = self._fetch_source(src_name, full=full) - stats["sources"][src_name] = new_count - total_new += new_count + try: + new_count = self._fetch_source(src_name, full=full) + stats["sources"][src_name] = new_count + total_new += new_count + except Exception as e: + console.print(f" [red]Error fetching {src_name}: {e}[/]") + stats["sources"][src_name] = {"error": str(e)} stats["new_docs"] = total_new console.print(f" Fetched [bold green]{total_new}[/] new documents total") # 3. Analyze unrated docs console.print("[bold]3/7[/] Analyzing unrated documents...") - analyzed = self.analyzer.rate_all_unrated(limit=200, batch_size=5) - stats["analyzed"] = analyzed + try: + analyzed = self.analyzer.rate_all_unrated(limit=200, batch_size=5) + stats["analyzed"] = analyzed + except Exception as e: + console.print(f" [red]Analysis failed: {e}[/]") + stats["analyzed"] = 0 + stats["errors"] = stats.get("errors", []) + [f"analyze: {e}"] # 4. Embed missing docs console.print("[bold]4/7[/] Embedding missing documents...") - embedded = self._embed_missing() - stats["embedded"] = embedded + try: + embedded = self._embed_missing() + stats["embedded"] = embedded + except Exception as e: + console.print(f" [red]Embedding failed: {e}[/]") + stats["embedded"] = 0 + stats["errors"] = stats.get("errors", []) + [f"embed: {e}"] # 5. Extract ideas from new docs console.print("[bold]5/7[/] Extracting ideas...") - ideas = self.analyzer.extract_all_ideas(limit=200, batch_size=5, cheap=True) - stats["ideas"] = ideas + try: + ideas = self.analyzer.extract_all_ideas(limit=200, batch_size=5, cheap=True) + stats["ideas"] = ideas + except Exception as e: + console.print(f" [red]Idea extraction failed: {e}[/]") + stats["ideas"] = 0 + stats["errors"] = stats.get("errors", []) + [f"ideas: {e}"] # 6. Re-run gap analysis if enough new docs if total_new >= 5: console.print("[bold]6/7[/] Re-running gap analysis...") - gaps = self.analyzer.gap_analysis() - if gaps: - self.db.record_gap_history(snapshot_id, gaps) - stats["gaps_changed"] = True - console.print(f" Found [bold]{len(gaps)}[/] gaps") + try: + gaps = self.analyzer.gap_analysis() + if gaps: + self.db.record_gap_history(snapshot_id, gaps) + stats["gaps_changed"] = True + console.print(f" Found [bold]{len(gaps)}[/] gaps") + except Exception as e: + console.print(f" [red]Gap analysis failed: {e}[/]") + stats["errors"] = stats.get("errors", []) + [f"gaps: {e}"] else: console.print(f"[bold]6/7[/] Skipping gap analysis ({total_new} < 5 new docs)") # Record current gaps unchanged @@ -142,7 +186,7 @@ class Observatory: self.db.upsert_source(src_name, doc_count=count) console.print("\n[bold green]Observatory update complete![/]") - console.print(f" New docs: {total_new} | Analyzed: {analyzed} | Embedded: {embedded} | Ideas: {ideas}") + console.print(f" New docs: {total_new} | Analyzed: {stats['analyzed']} | Embedded: {stats['embedded']} | Ideas: {stats['ideas']}") return stats def _fetch_source(self, source_name: str, full: bool = False) -> int: diff --git a/src/ietf_analyzer/readiness.py b/src/ietf_analyzer/readiness.py new file mode 100644 index 0000000..4af7353 --- /dev/null +++ b/src/ietf_analyzer/readiness.py @@ -0,0 +1,102 @@ +"""Standards readiness scoring — composite 0-100 score predicting RFC proximity.""" + +from __future__ import annotations + + +def compute_readiness(db, draft_name: str) -> dict: + """Compute 0-100 readiness score with component breakdown. + + Factors (each 0-1, weighted): + - wg_adopted (0.25): name starts with 'draft-ietf-' = 1.0, else 0.0 + - revision_maturity (0.15): int(rev) normalized (0-5+ maps to 0-1) + - reference_density (0.15): len(draft_refs) / max_refs across corpus + - cited_by_count (0.15): how many OTHER drafts in corpus reference this one + - author_experience (0.15): avg number of drafts per author in draft_authors + - momentum_rating (0.15): momentum score from ratings (1-5 -> 0-1) + + Returns {score: 0-100, factors: {name: {value, weight, contribution}}} + """ + draft = db.get_draft(draft_name) + if not draft: + return {"score": 0, "factors": {}} + + factors = {} + + # 1. WG Adopted (0.25) + wg_val = 1.0 if draft_name.startswith("draft-ietf-") else 0.0 + factors["wg_adopted"] = {"value": wg_val, "weight": 0.25, + "label": "WG Adopted", + "detail": "draft-ietf-*" if wg_val else "individual"} + + # 2. Revision Maturity (0.15) + try: + rev_num = int(draft.rev) if draft.rev else 0 + except (ValueError, TypeError): + rev_num = 0 + rev_val = min(rev_num / 5.0, 1.0) + factors["revision_maturity"] = {"value": round(rev_val, 3), "weight": 0.15, + "label": "Revision Maturity", + "detail": f"rev {rev_num}"} + + # 3. Reference Density (0.15) + refs = db.get_refs_for_draft(draft_name) + ref_count = len(refs) + # Get max refs across corpus + max_refs = db.conn.execute( + "SELECT MAX(cnt) FROM (SELECT COUNT(*) as cnt FROM draft_refs GROUP BY draft_name)" + ).fetchone()[0] or 1 + ref_val = min(ref_count / max_refs, 1.0) + factors["reference_density"] = {"value": round(ref_val, 3), "weight": 0.15, + "label": "Reference Density", + "detail": f"{ref_count} refs (max {max_refs})"} + + # 4. Cited By Count (0.15) + cited_by = db.conn.execute( + "SELECT COUNT(DISTINCT draft_name) FROM draft_refs WHERE ref_type = 'draft' AND ref_id = ?", + (draft_name,), + ).fetchone()[0] + # Normalize: being cited by 5+ drafts = 1.0 + cited_val = min(cited_by / 5.0, 1.0) + factors["cited_by_count"] = {"value": round(cited_val, 3), "weight": 0.15, + "label": "Cited By Others", + "detail": f"{cited_by} draft(s)"} + + # 5. Author Experience (0.15) + authors = db.get_authors_for_draft(draft_name) + if authors: + author_draft_counts = [] + for a in authors: + cnt = db.conn.execute( + "SELECT COUNT(*) FROM draft_authors WHERE person_id = ?", + (a.person_id,), + ).fetchone()[0] + author_draft_counts.append(cnt) + avg_exp = sum(author_draft_counts) / len(author_draft_counts) + # Normalize: avg 5+ drafts per author = 1.0 + exp_val = min(avg_exp / 5.0, 1.0) + else: + exp_val = 0.0 + avg_exp = 0 + factors["author_experience"] = {"value": round(exp_val, 3), "weight": 0.15, + "label": "Author Experience", + "detail": f"avg {avg_exp:.1f} drafts/author"} + + # 6. Momentum Rating (0.15) + rating = db.get_rating(draft_name) + if rating: + mom_val = (rating.momentum - 1) / 4.0 # 1-5 -> 0-1 + else: + mom_val = 0.0 + factors["momentum_rating"] = {"value": round(mom_val, 3), "weight": 0.15, + "label": "Momentum", + "detail": f"{rating.momentum}/5" if rating else "unrated"} + + # Compute weighted score + total = sum(f["value"] * f["weight"] for f in factors.values()) + score = round(total * 100, 1) + + # Add contribution to each factor + for f in factors.values(): + f["contribution"] = round(f["value"] * f["weight"] * 100, 1) + + return {"score": score, "factors": factors} diff --git a/src/ietf_analyzer/search.py b/src/ietf_analyzer/search.py new file mode 100644 index 0000000..cbff081 --- /dev/null +++ b/src/ietf_analyzer/search.py @@ -0,0 +1,280 @@ +"""Hybrid search — FTS5 keyword + embedding similarity + Claude synthesis.""" + +from __future__ import annotations + +import hashlib +from collections import defaultdict + +import numpy as np +from rich.console import Console + +from .config import Config +from .db import Database + +console = Console() + +ASK_PROMPT = """\ +You are an expert on IETF Internet-Drafts related to AI agents and autonomous systems. + +Based on the following IETF drafts, answer this question: + +**{question}** + +## Source Drafts + +{sources_block} + +Instructions: +- Answer concisely but thoroughly (3-8 sentences). +- Cite specific drafts by name (e.g. draft-xyz-...) when referencing their contributions. +- If the drafts don't contain enough information to answer, say so clearly. +- Focus on concrete technical mechanisms, not general observations.""" + + +def _cosine_similarity(a: np.ndarray, b: np.ndarray) -> float: + dot = np.dot(a, b) + norm = np.linalg.norm(a) * np.linalg.norm(b) + if norm == 0: + return 0.0 + return float(dot / norm) + + +def _prompt_hash(text: str) -> str: + return hashlib.sha256(text.encode()).hexdigest()[:16] + + +class HybridSearch: + def __init__(self, config: Config, db: Database, embedder=None): + self.config = config + self.db = db + self._embedder = embedder + self._ollama_available: bool | None = None + + @property + def embedder(self): + """Lazy-load embedder to avoid import errors when Ollama is unavailable.""" + if self._embedder is None: + try: + from .embeddings import Embedder + self._embedder = Embedder(self.config, self.db) + self._ollama_available = True + except Exception: + self._ollama_available = False + return self._embedder + + def _check_ollama(self) -> bool: + """Check if Ollama is available for embedding queries.""" + if self._ollama_available is not None: + return self._ollama_available + try: + embedder = self.embedder + if embedder is None: + self._ollama_available = False + return False + # Try a tiny embedding to verify connectivity + embedder.embed_text("test") + self._ollama_available = True + except Exception: + self._ollama_available = False + return self._ollama_available + + def search(self, query: str, top_k: int = 10) -> list[dict]: + """Combine FTS5 keyword search + embedding similarity search. + + Returns ranked list of {name, title, score, excerpt, match_type}. + Falls back to FTS5-only if Ollama is unavailable. + """ + fts_results = self._fts_search(query, limit=top_k * 2) + embed_results = self._embedding_search(query, limit=top_k * 2) + + if embed_results: + merged = self._reciprocal_rank_fusion(fts_results, embed_results) + else: + merged = fts_results + + return merged[:top_k] + + def _fts_search(self, query: str, limit: int = 20) -> list[dict]: + """Run FTS5 keyword search, return ranked results.""" + try: + drafts = self.db.search_drafts(query, limit=limit) + except Exception: + # FTS5 can fail on certain query syntax; fallback to simpler search + # Try wrapping each word with quotes for literal matching + words = query.split() + safe_query = " OR ".join(f'"{w}"' for w in words if w.strip()) + try: + drafts = self.db.search_drafts(safe_query, limit=limit) + except Exception: + return [] + + results = [] + for rank, draft in enumerate(drafts): + excerpt = draft.abstract[:200] if draft.abstract else "" + results.append({ + "name": draft.name, + "title": draft.title, + "score": 0.0, # Will be set by fusion + "excerpt": excerpt, + "match_type": "keyword", + "rank": rank, + }) + return results + + def _embedding_search(self, query: str, limit: int = 20) -> list[dict]: + """Run embedding similarity search against all stored embeddings.""" + if not self._check_ollama(): + return [] + + try: + query_vec = self.embedder.embed_text(query) + except Exception: + self._ollama_available = False + return [] + + all_embeddings = self.db.all_embeddings() + if not all_embeddings: + return [] + + similarities: list[tuple[str, float]] = [] + for name, vec in all_embeddings.items(): + sim = _cosine_similarity(query_vec, vec) + similarities.append((name, sim)) + + similarities.sort(key=lambda x: x[1], reverse=True) + + results = [] + for rank, (name, sim) in enumerate(similarities[:limit]): + draft = self.db.get_draft(name) + if draft is None: + continue + excerpt = draft.abstract[:200] if draft.abstract else "" + results.append({ + "name": name, + "title": draft.title if draft else name, + "score": round(sim, 4), + "excerpt": excerpt, + "match_type": "semantic", + "rank": rank, + "similarity": round(sim, 4), + }) + return results + + def _reciprocal_rank_fusion( + self, + fts_results: list[dict], + embed_results: list[dict], + k: int = 60, + ) -> list[dict]: + """Merge two ranked lists using reciprocal rank fusion (RRF). + + RRF score = sum(1 / (k + rank)) across all lists where the item appears. + """ + scores: dict[str, float] = defaultdict(float) + items: dict[str, dict] = {} + + for result in fts_results: + name = result["name"] + scores[name] += 1.0 / (k + result["rank"]) + if name not in items: + items[name] = result.copy() + items[name]["match_type"] = "keyword" + + for result in embed_results: + name = result["name"] + scores[name] += 1.0 / (k + result["rank"]) + if name in items: + items[name]["match_type"] = "both" + items[name]["similarity"] = result.get("similarity", 0) + else: + items[name] = result.copy() + + # Sort by RRF score + ranked = sorted(scores.items(), key=lambda x: x[1], reverse=True) + + results = [] + for name, rrf_score in ranked: + item = items[name] + item["score"] = round(rrf_score, 4) + results.append(item) + + return results + + def ask(self, question: str, top_k: int = 5, cheap: bool = True) -> dict: + """Answer a natural language question using search + Claude synthesis. + + Returns {answer: str, sources: [{name, title, similarity, excerpt}]}. + Caches Claude responses via llm_cache. + """ + search_results = self.search(question, top_k=top_k) + + if not search_results: + return { + "answer": "No relevant drafts found for your question.", + "sources": [], + } + + # Build context from top results + sources_block = "" + sources = [] + for r in search_results: + draft = self.db.get_draft(r["name"]) + if draft is None: + continue + + # Title + abstract + first 500 chars of full text + text_preview = "" + if draft.full_text: + text_preview = draft.full_text[:500] + + sources_block += f"\n---\n**{draft.name}** — {draft.title}\n" + sources_block += f"Abstract: {draft.abstract[:500]}\n" + if text_preview: + sources_block += f"Content excerpt: {text_preview}\n" + + sources.append({ + "name": draft.name, + "title": draft.title, + "similarity": r.get("similarity", r.get("score", 0)), + "excerpt": r.get("excerpt", ""), + "match_type": r.get("match_type", ""), + }) + + prompt = ASK_PROMPT.format( + question=question, + sources_block=sources_block, + ) + phash = _prompt_hash(prompt) + + # Check cache + cached = self.db.get_cached_response("_ask_", phash) + if cached: + return { + "answer": cached, + "sources": sources, + } + + # Call Claude + try: + from .analyzer import Analyzer + analyzer = Analyzer(self.config, self.db) + text, in_tok, out_tok = analyzer._call_claude( + prompt, max_tokens=1024, cheap=cheap + ) + + # Cache the response + self.db.cache_response( + "_ask_", phash, + self.config.claude_model_cheap if cheap else self.config.claude_model, + prompt, text, in_tok, out_tok, + ) + + return { + "answer": text, + "sources": sources, + } + except Exception as e: + return { + "answer": f"Error generating answer: {e}", + "sources": sources, + } diff --git a/src/webui/app.py b/src/webui/app.py index 78e76f1..25d77a7 100644 --- a/src/webui/app.py +++ b/src/webui/app.py @@ -12,7 +12,11 @@ from pathlib import Path _project_root = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(_project_root / "src")) -from flask import Flask, render_template, request, jsonify, abort, g +import csv +import io +import json + +from flask import Flask, render_template, request, jsonify, abort, g, Response from webui.data import ( get_db, @@ -39,6 +43,10 @@ from webui.data import ( get_idea_clusters, get_monitor_status, get_author_network_full, + get_citation_graph, + get_comparison_data, + get_ask_data, + global_search, ) app = Flask( @@ -91,6 +99,7 @@ def drafts(): page = request.args.get("page", 1, type=int) search = request.args.get("q", "") category = request.args.get("cat", "") + source = request.args.get("source", "") min_score = request.args.get("min_score", 0.0, type=float) sort = request.args.get("sort", "score") sort_dir = request.args.get("dir", "desc") @@ -103,6 +112,7 @@ def drafts(): min_score=min_score, sort=sort, sort_dir=sort_dir, + source=source, ) categories = get_category_counts(db()) return render_template( @@ -111,6 +121,7 @@ def drafts(): categories=categories, search=search, current_cat=category, + current_source=source, min_score=min_score, sort=sort, sort_dir=sort_dir, @@ -272,6 +283,12 @@ def authors(): ) +@app.route("/citations") +def citations(): + graph = get_citation_graph(db()) + return render_template("citations.html", graph=graph) + + @app.route("/monitor") def monitor_page(): status = get_monitor_status(db()) @@ -294,21 +311,121 @@ def datenschutz(): return render_template("datenschutz.html") +@app.route("/search") +def search(): + q = request.args.get("q", "").strip() + results = global_search(db(), q) if q else {"drafts": [], "ideas": [], "authors": [], "gaps": []} + total = sum(len(v) for v in results.values()) + return render_template("search_results.html", query=q, results=results, total=total) + + +@app.route("/ask") +def ask_page(): + question = request.args.get("q", "") + result = None + if question: + top_k = request.args.get("top", 5, type=int) + result = get_ask_data(db(), question, top_k=top_k) + return render_template("ask.html", question=question, result=result) + + +@app.route("/api/ask", methods=["POST"]) +def api_ask(): + """Answer a question via hybrid search + Claude. Returns JSON.""" + data = request.get_json(force=True, silent=True) + if not data or "question" not in data: + return jsonify({"error": "Missing 'question' in request body"}), 400 + question = data["question"] + top_k = data.get("top_k", 5) + cheap = data.get("cheap", True) + result = get_ask_data(db(), question, top_k=top_k, cheap=cheap) + return jsonify(result) + + +@app.route("/compare") +def compare_page(): + draft_names = request.args.get("drafts", "") + names = [n.strip() for n in draft_names.split(",") if n.strip()] if draft_names else [] + data = None + if len(names) >= 2: + data = get_comparison_data(db(), names) + return render_template("comparison.html", names=names, data=data) + + +@app.route("/api/compare", methods=["POST"]) +def api_compare(): + """Run Claude comparison for drafts. Returns JSON with comparison text.""" + req_data = request.get_json(force=True, silent=True) + if not req_data or "drafts" not in req_data: + return jsonify({"error": "Missing 'drafts' in request body"}), 400 + + names = req_data["drafts"] + if len(names) < 2: + return jsonify({"error": "Need at least 2 drafts to compare"}), 400 + + try: + from ietf_analyzer.config import Config + from ietf_analyzer.analyzer import Analyzer + + cfg = Config.load() + database = db() + analyzer = Analyzer(cfg, database) + result = analyzer.compare_drafts(names) + return jsonify(result) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + # --- API endpoints for AJAX (used by client-side charts) --- +def _to_csv_response(rows: list[dict], filename: str = "export.csv") -> Response: + """Convert a list of dicts to a CSV download response.""" + if not rows: + return Response("", mimetype="text/csv", + headers={"Content-Disposition": f"attachment; filename={filename}"}) + si = io.StringIO() + writer = csv.DictWriter(si, fieldnames=rows[0].keys()) + writer.writeheader() + for row in rows: + # Flatten any list/dict values to JSON strings + flat = {} + for k, v in row.items(): + if isinstance(v, (list, dict)): + flat[k] = json.dumps(v) + else: + flat[k] = v + writer.writerow(flat) + return Response(si.getvalue(), mimetype="text/csv", + headers={"Content-Disposition": f"attachment; filename={filename}"}) + + +def _results_to_csv(results: dict) -> Response: + """Convert global search results (multi-category) to a single CSV.""" + rows = [] + for category, items in results.items(): + for item in items: + row = {"_category": category} + row.update(item) + rows.append(row) + return _to_csv_response(rows, "search_results.csv") + + @app.route("/api/drafts") def api_drafts(): page = request.args.get("page", 1, type=int) search = request.args.get("q", "") category = request.args.get("cat", "") + source = request.args.get("source", "") min_score = request.args.get("min_score", 0.0, type=float) sort = request.args.get("sort", "score") sort_dir = request.args.get("dir", "desc") - return jsonify( - get_drafts_page(db(), page=page, search=search, category=category, - min_score=min_score, sort=sort, sort_dir=sort_dir) - ) + data = get_drafts_page(db(), page=page, search=search, category=category, + min_score=min_score, sort=sort, sort_dir=sort_dir, + source=source) + if request.args.get("format") == "csv": + return _to_csv_response(data.get("drafts", []), "drafts.csv") + return jsonify(data) @app.route("/api/stats") @@ -321,6 +438,148 @@ def api_author_network(): return jsonify(get_author_network_full(db())) +@app.route("/api/citations") +def api_citations(): + min_refs = request.args.get("min_refs", 2, type=int) + return jsonify(get_citation_graph(db(), min_refs=min_refs)) + + +@app.route("/api/search") +def api_search(): + q = request.args.get("q", "").strip() + results = global_search(db(), q) if q else {"drafts": [], "ideas": [], "authors": [], "gaps": []} + if request.args.get("format") == "csv": + return _results_to_csv(results) + return jsonify(results) + + +@app.route("/api/ideas") +def api_ideas(): + data = get_ideas_by_type(db()) + if request.args.get("format") == "csv": + return _to_csv_response(data.get("ideas", []), "ideas.csv") + return jsonify(data) + + +@app.route("/api/gaps") +def api_gaps(): + data = get_all_gaps(db()) + if request.args.get("format") == "csv": + return _to_csv_response(data, "gaps.csv") + return jsonify(data) + + +@app.route("/api/gaps/") +def api_gap_detail(gap_id: int): + gap = get_gap_detail(db(), gap_id) + if not gap: + return jsonify({"error": "Gap not found"}), 404 + return jsonify(gap) + + +@app.route("/api/ratings") +def api_ratings(): + data = get_rating_distributions(db()) + if request.args.get("format") == "csv": + # Transpose columnar data to rows + rows = [] + for i in range(len(data.get("names", []))): + rows.append({ + "name": data["names"][i], + "score": data["scores"][i], + "novelty": data["novelty"][i], + "maturity": data["maturity"][i], + "overlap": data["overlap"][i], + "momentum": data["momentum"][i], + "relevance": data["relevance"][i], + "category": data["categories"][i], + }) + return _to_csv_response(rows, "ratings.csv") + return jsonify(data) + + +@app.route("/api/timeline") +def api_timeline(): + data = get_timeline_data(db()) + return jsonify(data) + + +@app.route("/api/landscape") +def api_landscape(): + data = get_landscape_tsne(db()) + if request.args.get("format") == "csv": + return _to_csv_response(data, "landscape.csv") + return jsonify(data) + + +@app.route("/api/similarity") +def api_similarity(): + data = get_similarity_graph(db()) + return jsonify(data) + + +@app.route("/api/idea-clusters") +def api_idea_clusters(): + data = get_idea_clusters(db()) + return jsonify(data) + + +@app.route("/api/monitor") +def api_monitor(): + data = get_monitor_status(db()) + return jsonify(data) + + +@app.route("/api/drafts/") +def api_draft_detail(name: str): + detail = get_draft_detail(db(), name) + if not detail: + return jsonify({"error": "Draft not found"}), 404 + return jsonify(detail) + + +@app.route("/api/categories") +def api_categories(): + data = get_category_counts(db()) + if request.args.get("format") == "csv": + rows = [{"category": k, "count": v} for k, v in data.items()] + return _to_csv_response(rows, "categories.csv") + return jsonify(data) + + +@app.route("/api/drafts//annotate", methods=["POST"]) +def api_annotate(name: str): + """Add or update annotation for a draft.""" + import json as _json + database = db() + draft = database.get_draft(name) + if not draft: + return jsonify({"error": "Draft not found"}), 404 + + data = request.get_json(force=True, silent=True) + if not data: + return jsonify({"error": "Invalid JSON body"}), 400 + + note = data.get("note") + tags = data.get("tags") + add_tag = data.get("add_tag") + remove_tag = data.get("remove_tag") + + # Handle add/remove tag operations + if add_tag or remove_tag: + existing = database.get_annotation(name) + current_tags = existing["tags"] if existing else [] + if add_tag and add_tag not in current_tags: + current_tags.append(add_tag) + if remove_tag and remove_tag in current_tags: + current_tags.remove(remove_tag) + tags = current_tags + + database.upsert_annotation(name, note=note, tags=tags) + annotation = database.get_annotation(name) + return jsonify({"success": True, "annotation": annotation}) + + if __name__ == "__main__": print("Starting IETF Draft Analyzer Dashboard on http://127.0.0.1:5000") app.run(debug=True, host="127.0.0.1", port=5000) diff --git a/src/webui/data.py b/src/webui/data.py index 8f600e5..0307e4d 100644 --- a/src/webui/data.py +++ b/src/webui/data.py @@ -66,6 +66,7 @@ def get_drafts_page( min_score: float = 0.0, sort: str = "score", sort_dir: str = "desc", + source: str = "", ) -> dict: """Return a paginated, filtered list of drafts with ratings. @@ -80,6 +81,8 @@ def get_drafts_page( continue if category and category not in rating.categories: continue + if source and draft.source != source: + continue if search: haystack = f"{draft.name} {draft.title} {rating.summary}".lower() if not all(w in haystack for w in search.lower().split()): @@ -96,6 +99,9 @@ def get_drafts_page( "relevance": lambda p: p[1].relevance, "overlap": lambda p: p[1].overlap, "momentum": lambda p: p[1].momentum, + "readiness": lambda p: (1.0 if p[0].name.startswith("draft-ietf-") else 0.0) * 0.25 + + min(int(p[0].rev or "0") / 5.0, 1.0) * 0.15 + + ((p[1].momentum - 1) / 4.0) * 0.15, } key_fn = sort_keys.get(sort, sort_keys["score"]) reverse = sort_dir == "desc" @@ -107,15 +113,23 @@ def get_drafts_page( start = (page - 1) * per_page page_items = filtered[start : start + per_page] + # Pre-compute readiness for page items (lightweight version) + from ietf_analyzer.readiness import compute_readiness + readiness_cache = {} + for draft, rating in page_items: + readiness_cache[draft.name] = compute_readiness(db, draft.name) + drafts = [] for draft, rating in page_items: + r_score = readiness_cache.get(draft.name, {}).get("score", 0) drafts.append({ "name": draft.name, "title": draft.title, "date": draft.date, - "url": draft.datatracker_url, + "url": draft.source_url if draft.source != "ietf" else draft.datatracker_url, "pages": draft.pages or 0, "group": draft.group or "individual", + "source": draft.source or "ietf", "score": round(rating.composite_score, 2), "novelty": rating.novelty, "maturity": rating.maturity, @@ -124,6 +138,7 @@ def get_drafts_page( "relevance": rating.relevance, "categories": rating.categories, "summary": rating.summary, + "readiness": r_score, }) return { @@ -185,6 +200,14 @@ def get_draft_detail(db: Database, name: str) -> dict | None: "categories": rating.categories, } + # Readiness score + from ietf_analyzer.readiness import compute_readiness + result["readiness"] = compute_readiness(db, name) + + # Annotation + annotation = db.get_annotation(name) + result["annotation"] = annotation + return result @@ -253,8 +276,11 @@ def get_ideas_by_type(db: Database) -> dict: def get_all_gaps(db: Database) -> list[dict]: - """Return all gap analysis results.""" - return db.all_gaps() + """Return all gap analysis results, sorted by severity (critical first).""" + _sev_order = {"critical": 0, "high": 1, "medium": 2, "low": 3} + gaps = db.all_gaps() + gaps.sort(key=lambda g: _sev_order.get(g.get("severity", "low"), 99)) + return gaps def get_gap_detail(db: Database, gap_id: int) -> dict | None: @@ -775,17 +801,252 @@ def get_monitor_status(db: Database) -> dict: """Return monitoring status data for dashboard.""" runs = db.get_monitor_runs(limit=20) last = runs[0] if runs else None + total_drafts = db.count_drafts() + rated_count = len(db.drafts_with_ratings(limit=10000)) unrated = len(db.unrated_drafts(limit=9999)) unembedded = len(db.drafts_without_embeddings(limit=9999)) + embedded_count = total_drafts - unembedded no_ideas = len(db.drafts_without_ideas(limit=9999)) + ideas_count = total_drafts - no_ideas + idea_total = db.idea_count() + gap_count = len(db.all_gaps()) + input_tok, output_tok = db.total_tokens_used() + + # Estimate cost (Sonnet pricing: $3/M input, $15/M output) + est_cost = (input_tok * 3.0 / 1_000_000) + (output_tok * 15.0 / 1_000_000) + return { "last_run": last, "runs": runs, "unprocessed": {"unrated": unrated, "unembedded": unembedded, "no_ideas": no_ideas}, "total_runs": len(runs), + "pipeline": { + "total_drafts": total_drafts, + "rated": rated_count, + "embedded": embedded_count, + "with_ideas": ideas_count, + "idea_total": idea_total, + "gap_count": gap_count, + }, + "cost": { + "input_tokens": input_tok, + "output_tokens": output_tok, + "estimated_usd": round(est_cost, 2), + }, } +def get_citation_graph(db: Database, min_refs: int = 2) -> dict: + """Return citation network data for force-directed graph. + + Returns {nodes: [{id, type, title, influence, ...}], + edges: [{source, target}], + stats: {node_count, edge_count, ...}} + """ + # Get all references + rows = db.conn.execute( + "SELECT draft_name, ref_type, ref_id FROM draft_refs" + ).fetchall() + + # Count in-degree for each referenced item + in_degree: dict[str, int] = Counter() + edges_raw = [] + for r in rows: + ref_key = f"{r['ref_type']}:{r['ref_id']}" + in_degree[ref_key] += 1 + edges_raw.append((r["draft_name"], ref_key)) + + # Also count drafts as source nodes + draft_out: dict[str, int] = Counter() + for draft_name, _ in edges_raw: + draft_out[draft_name] += 1 + + # Get draft titles for labeling + draft_rows = db.conn.execute("SELECT name, title FROM drafts").fetchall() + draft_titles = {r["name"]: r["title"] for r in draft_rows} + + # Get rating categories for draft coloring + rating_rows = db.conn.execute("SELECT draft_name, categories FROM ratings").fetchall() + draft_cats = {} + for r in rating_rows: + try: + cats = json.loads(r["categories"]) if r["categories"] else [] + draft_cats[r["draft_name"]] = cats[0] if cats else "Other" + except Exception: + draft_cats[r["draft_name"]] = "Other" + + # Filter: keep RFCs with min_refs+ references and all drafts that reference them + top_refs = {k: v for k, v in in_degree.items() if v >= min_refs} + + # Build node set + node_set = set() + filtered_edges = [] + for draft_name, ref_key in edges_raw: + if ref_key in top_refs: + node_set.add(draft_name) + node_set.add(ref_key) + filtered_edges.append({"source": draft_name, "target": ref_key}) + + # Limit to ~200 nodes max for readability + if len(node_set) > 250: + # Keep only refs with higher in-degree + sorted_refs = sorted(top_refs.items(), key=lambda x: x[1], reverse=True) + keep_refs = set(k for k, _ in sorted_refs[:80]) + node_set = set() + filtered_edges = [] + for draft_name, ref_key in edges_raw: + if ref_key in keep_refs: + node_set.add(draft_name) + node_set.add(ref_key) + filtered_edges.append({"source": draft_name, "target": ref_key}) + + # Build nodes + nodes = [] + for nid in node_set: + if ":" in nid and not nid.startswith("draft-"): + # It's a reference node (rfc:1234, bcp:14, etc.) + ref_type, ref_id = nid.split(":", 1) + influence = in_degree.get(nid, 0) + if ref_type == "rfc": + try: + title = f"RFC {int(ref_id)}" + except ValueError: + title = f"RFC {ref_id}" + else: + title = f"{ref_type.upper()} {ref_id}" + nodes.append({ + "id": nid, + "type": ref_type, + "title": title, + "influence": influence, + "ref_id": ref_id, + }) + else: + # It's a draft node + influence = in_degree.get(nid, 0) + draft_out.get(nid, 0) + nodes.append({ + "id": nid, + "type": "draft", + "title": draft_titles.get(nid, nid), + "influence": draft_out.get(nid, 0), + "category": draft_cats.get(nid, "Other"), + }) + + # Stats + rfc_count = sum(1 for n in nodes if n["type"] == "rfc") + draft_count = sum(1 for n in nodes if n["type"] == "draft") + + return { + "nodes": nodes, + "edges": filtered_edges, + "stats": { + "node_count": len(nodes), + "edge_count": len(filtered_edges), + "rfc_count": rfc_count, + "draft_count": draft_count, + }, + } + + +def global_search(db: Database, query: str) -> dict: + """Search across drafts (FTS5), ideas, authors, and gaps. + + Returns {drafts: [...], ideas: [...], authors: [...], gaps: [...]}. + """ + results: dict = {"drafts": [], "ideas": [], "authors": [], "gaps": []} + if not query or not query.strip(): + return results + + q = query.strip() + + # 1. Drafts via FTS5 + try: + fts_query = " ".join(f'"{w}"' for w in q.split() if w) + rows = db.conn.execute( + """SELECT d.name, d.title, d.abstract, d.time, d."group" + FROM drafts d + JOIN drafts_fts f ON d.rowid = f.rowid + WHERE drafts_fts MATCH ? + ORDER BY rank + LIMIT 50""", + (fts_query,), + ).fetchall() + for r in rows: + results["drafts"].append({ + "name": r["name"], + "title": r["title"], + "abstract": (r["abstract"] or "")[:200], + "date": r["time"], + "group": r["group"] or "individual", + }) + except Exception: + # FTS5 match can fail on certain query syntax; fall back to LIKE + like = f"%{q}%" + rows = db.conn.execute( + """SELECT name, title, abstract, time, "group" FROM drafts + WHERE title LIKE ? OR name LIKE ? OR abstract LIKE ? + LIMIT 50""", + (like, like, like), + ).fetchall() + for r in rows: + results["drafts"].append({ + "name": r["name"], + "title": r["title"], + "abstract": (r["abstract"] or "")[:200], + "date": r["time"], + "group": r["group"] or "individual", + }) + + # 2. Ideas via LIKE + like = f"%{q}%" + rows = db.conn.execute( + """SELECT id, title, description, idea_type, draft_name FROM ideas + WHERE title LIKE ? OR description LIKE ? + ORDER BY id LIMIT 50""", + (like, like), + ).fetchall() + for r in rows: + results["ideas"].append({ + "id": r["id"], + "title": r["title"], + "description": (r["description"] or "")[:200], + "type": r["idea_type"], + "draft_name": r["draft_name"], + }) + + # 3. Authors via LIKE + rows = db.conn.execute( + """SELECT person_id, name, affiliation FROM authors + WHERE name LIKE ? OR affiliation LIKE ? + ORDER BY name LIMIT 50""", + (like, like), + ).fetchall() + for r in rows: + results["authors"].append({ + "person_id": r["person_id"], + "name": r["name"], + "affiliation": r["affiliation"] or "", + }) + + # 4. Gaps via LIKE + rows = db.conn.execute( + """SELECT id, topic, description, category, severity FROM gaps + WHERE topic LIKE ? OR description LIKE ? + ORDER BY id LIMIT 50""", + (like, like), + ).fetchall() + for r in rows: + results["gaps"].append({ + "id": r["id"], + "topic": r["topic"], + "description": (r["description"] or "")[:200], + "category": r["category"], + "severity": r["severity"], + }) + + return results + + def get_landscape_tsne(db: Database) -> list[dict]: """Compute t-SNE from embeddings, return [{name, title, x, y, category, score}]. @@ -829,3 +1090,116 @@ def get_landscape_tsne(db: Database) -> list[dict]: "score": round(r.composite_score, 2), }) return result + + +def get_comparison_data(db: Database, names: list[str]) -> dict | None: + """Get comparison data for a list of drafts. + + Returns { + drafts: [{name, title, abstract, rating, ideas, refs, ...}], + shared_ideas: [{title, drafts: [name,...]}], + unique_ideas: {name: [{title, description}]}, + shared_refs: [{type, id, drafts: [name,...]}], + unique_refs: {name: [{type, id}]}, + similarities: [{a, b, similarity}], + comparison_text: str | None, + } + """ + import numpy as np + + drafts_data = [] + all_ideas: dict[str, list[dict]] = {} + all_refs: dict[str, list[tuple[str, str]]] = {} + + for name in names: + detail = get_draft_detail(db, name) + if not detail: + continue + drafts_data.append(detail) + all_ideas[name] = detail.get("ideas", []) + all_refs[name] = [(r["type"], r["id"]) for r in detail.get("refs", [])] + + if len(drafts_data) < 2: + return None + + # Find shared vs unique ideas (by title similarity) + idea_title_drafts: dict[str, list[str]] = {} + for name, ideas in all_ideas.items(): + for idea in ideas: + title_lower = idea["title"].lower().strip() + if title_lower not in idea_title_drafts: + idea_title_drafts[title_lower] = [] + idea_title_drafts[title_lower].append(name) + + shared_ideas = [ + {"title": title, "drafts": draft_list} + for title, draft_list in idea_title_drafts.items() + if len(set(draft_list)) > 1 + ] + unique_ideas: dict[str, list[dict]] = {} + for name, ideas in all_ideas.items(): + unique = [] + for idea in ideas: + title_lower = idea["title"].lower().strip() + if len(set(idea_title_drafts.get(title_lower, []))) <= 1: + unique.append({"title": idea["title"], "description": idea.get("description", "")}) + unique_ideas[name] = unique + + # Find shared vs unique references + ref_drafts: dict[tuple[str, str], list[str]] = {} + for name, refs in all_refs.items(): + for ref in refs: + if ref not in ref_drafts: + ref_drafts[ref] = [] + ref_drafts[ref].append(name) + + shared_refs = [ + {"type": ref[0], "id": ref[1], "drafts": draft_list} + for ref, draft_list in ref_drafts.items() + if len(set(draft_list)) > 1 + ] + unique_refs: dict[str, list[dict]] = {} + for name, refs in all_refs.items(): + unique = [] + for ref in refs: + if len(set(ref_drafts.get(ref, []))) <= 1: + unique.append({"type": ref[0], "id": ref[1]}) + unique_refs[name] = unique + + # Pairwise embedding similarities + embeddings = db.all_embeddings() + similarities = [] + valid_names = [d["name"] for d in drafts_data] + for i in range(len(valid_names)): + for j in range(i + 1, len(valid_names)): + a, b = valid_names[i], valid_names[j] + if a in embeddings and b in embeddings: + vec_a = embeddings[a] + vec_b = embeddings[b] + dot = np.dot(vec_a, vec_b) + norm = np.linalg.norm(vec_a) * np.linalg.norm(vec_b) + sim = float(dot / norm) if norm > 0 else 0.0 + similarities.append({"a": a, "b": b, "similarity": round(sim, 4)}) + + return { + "drafts": drafts_data, + "shared_ideas": shared_ideas, + "unique_ideas": unique_ideas, + "shared_refs": shared_refs, + "unique_refs": unique_refs, + "similarities": similarities, + "comparison_text": None, + } + + +def get_ask_data(db: Database, question: str, top_k: int = 5, cheap: bool = True) -> dict: + """Run hybrid search + Claude synthesis for a question. + + Returns {answer: str, sources: [{name, title, similarity, excerpt}]}. + """ + from ietf_analyzer.config import Config + from ietf_analyzer.search import HybridSearch + + config = Config.load() + searcher = HybridSearch(config, db) + return searcher.ask(question, top_k=top_k, cheap=cheap) diff --git a/src/webui/templates/ask.html b/src/webui/templates/ask.html new file mode 100644 index 0000000..9e3f571 --- /dev/null +++ b/src/webui/templates/ask.html @@ -0,0 +1,153 @@ +{% extends "base.html" %} +{% set active_page = "ask" %} + +{% block title %}Ask — IETF Draft Analyzer{% endblock %} + +{% block extra_head %} + +{% endblock %} + +{% block content %} + +
+

Ask the Draft Corpus

+

Ask natural language questions about IETF AI/agent drafts. Answers are synthesized from the most relevant documents.

+
+ + +
+
+
+ + + + + +
+
+ +
Combines keyword search + semantic similarity
+
+
+
+ + +{% if not question %} +
+
Example questions
+
+ {% set examples = [ + "Which drafts address agent authentication and identity?", + "What are the competing approaches to agent-to-agent communication?", + "How do safety mechanisms work across different proposals?", + "What protocols exist for AI model serving and inference?", + "Which drafts propose agent discovery or registration systems?", + "What are the main gaps in autonomous network operations?", + ] %} + {% for q in examples %} + + {{ q }} + + {% endfor %} +
+
+{% endif %} + + +{% if result %} +
+ +
+
+ + + +

Answer

+
+
{{ result.answer }}
+
+ + + {% if result.sources %} +
+
+

Source Drafts ({{ result.sources|length }})

+
+ + + + + + + + + + + {% for src in result.sources %} + + + + + + + {% endfor %} + +
#DraftMatchScore
{{ loop.index }} + + {{ src.title }} + +
{{ src.name }}
+ 		+ {% if src.match_type == 'both' %} + both + {% elif src.match_type == 'semantic' %} + semantic + {% else %} + keyword + {% endif %} + + {{ "%.3f"|format(src.similarity) if src.similarity else "-" }} +
+
+ {% endif %} +
+{% endif %} +{% endblock %} diff --git a/src/webui/templates/base.html b/src/webui/templates/base.html index 8618463..3c3cda7 100644 --- a/src/webui/templates/base.html +++ b/src/webui/templates/base.html @@ -82,11 +82,26 @@

IETF Draft Analyzer

AI/Agent Standards Tracker

+ +
+
+ + + + +
+