v0.2.0: visualizations, interactive browser, arXiv paper, gap analysis
New features: - 12 interactive visualizations (ietf viz): t-SNE landscape, similarity heatmap, score distributions, timeline, bubble explorer, radar charts, author network graph, category treemap, quality vs overlap, org bar chart, ideas chart, and interactive draft browser - Interactive draft browser (browser.html): filterable by category, keyword, score sliders with sortable table and expandable detail rows - arXiv paper (paper/main.tex): 13-page manuscript with all findings - Gap analysis: 12 identified under-addressed areas - Author network: collaboration graph, org contributions, cross-org analysis - Draft generation from gaps (ietf draft-gen) - Auto-load .env for API keys (python-dotenv) New modules: visualize.py, authors.py, draftgen.py New reports: timeline, overlap-matrix, authors, gaps New deps: plotly, matplotlib, seaborn, scipy, scikit-learn, networkx, python-dotenv Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,12 @@ from __future__ import annotations
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
# Load .env from project root (two levels up from this file, or cwd)
|
||||
load_dotenv(Path(__file__).resolve().parent.parent.parent / ".env")
|
||||
load_dotenv() # Also check cwd
|
||||
|
||||
import anthropic
|
||||
from rich.console import Console
|
||||
@@ -62,6 +68,53 @@ Compare these IETF drafts — overlaps, unique ideas, complementary vs competing
|
||||
|
||||
Be specific about concrete mechanisms and design choices."""
|
||||
|
||||
EXTRACT_IDEAS_PROMPT = """\
|
||||
Extract discrete technical ideas and mechanisms from this IETF draft.
|
||||
Return a JSON array. Each element: {{"title":"short name","description":"1-2 sentences","type":"mechanism|protocol|pattern|requirement|architecture|extension"}}
|
||||
|
||||
{name} | {title} | {pages}pg
|
||||
Abstract: {abstract}
|
||||
|
||||
{text_excerpt}
|
||||
|
||||
Return 3-8 ideas. Focus on CONCRETE technical contributions, not general statements.
|
||||
JSON array only, no fences."""
|
||||
|
||||
BATCH_IDEAS_PROMPT = """\
|
||||
Extract ideas from each IETF draft below. Return a JSON object mapping draft name -> array of ideas.
|
||||
Per idea: {{"title":"short name","description":"1 sentence","type":"mechanism|protocol|pattern|requirement|architecture|extension"}}
|
||||
|
||||
{drafts_block}
|
||||
|
||||
3-8 ideas per draft. CONCRETE technical contributions only.
|
||||
Return ONLY a JSON object like {{"draft-name":[...], ...}}, no fences."""
|
||||
|
||||
GAP_ANALYSIS_PROMPT = """\
|
||||
You are analyzing the landscape of {total} IETF Internet-Drafts related to AI agents and autonomous systems.
|
||||
|
||||
## Categories and Draft Counts
|
||||
{category_summary}
|
||||
|
||||
## Most Common Technical Ideas
|
||||
{top_ideas}
|
||||
|
||||
## Known Overlap Clusters (groups of highly similar drafts)
|
||||
{overlap_summary}
|
||||
|
||||
Identify 8-15 GAPS — areas, problems, or technical challenges NOT adequately addressed by existing drafts.
|
||||
|
||||
Return a JSON array:
|
||||
[{{"topic":"short topic name","description":"2-3 sentence description","category":"closest category or new","severity":"critical|high|medium|low","evidence":"what suggests this gap matters"}}]
|
||||
|
||||
Focus on:
|
||||
1. Problems mentioned but not solved
|
||||
2. Missing infrastructure pieces
|
||||
3. Security/privacy/safety issues not addressed
|
||||
4. Interoperability gaps between competing proposals
|
||||
5. Real-world deployment concerns ignored
|
||||
|
||||
JSON array only, no fences."""
|
||||
|
||||
|
||||
def _prompt_hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode()).hexdigest()[:16]
|
||||
@@ -100,10 +153,15 @@ class Analyzer:
|
||||
rated_at=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
|
||||
def _call_claude(self, prompt: str, max_tokens: int = 512) -> tuple[str, int, int]:
|
||||
"""Call Claude and return (text, input_tokens, output_tokens)."""
|
||||
def _call_claude(self, prompt: str, max_tokens: int = 512, cheap: bool = False) -> tuple[str, int, int]:
|
||||
"""Call Claude and return (text, input_tokens, output_tokens).
|
||||
|
||||
Args:
|
||||
cheap: If True, use claude_model_cheap (Haiku) for lower cost.
|
||||
"""
|
||||
model = self.config.claude_model_cheap if cheap else self.config.claude_model
|
||||
resp = self.client.messages.create(
|
||||
model=self.config.claude_model,
|
||||
model=model,
|
||||
max_tokens=max_tokens,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
)
|
||||
@@ -252,6 +310,232 @@ class Analyzer:
|
||||
)
|
||||
return count
|
||||
|
||||
def extract_ideas(self, draft_name: str, use_cache: bool = True) -> list[dict] | None:
|
||||
"""Extract technical ideas from a single draft."""
|
||||
draft = self.db.get_draft(draft_name)
|
||||
if draft is None:
|
||||
console.print(f"[red]Draft not found: {draft_name}[/]")
|
||||
return None
|
||||
|
||||
text_excerpt = ""
|
||||
if draft.full_text:
|
||||
text_excerpt = draft.full_text[:3000]
|
||||
|
||||
prompt = EXTRACT_IDEAS_PROMPT.format(
|
||||
name=draft.name, title=draft.title,
|
||||
pages=draft.pages or "?",
|
||||
abstract=draft.abstract[:2000],
|
||||
text_excerpt=text_excerpt,
|
||||
)
|
||||
phash = _prompt_hash("ideas-" + prompt)
|
||||
|
||||
if use_cache:
|
||||
cached = self.db.get_cached_response(draft_name, phash)
|
||||
if cached:
|
||||
try:
|
||||
ideas = json.loads(cached)
|
||||
if isinstance(ideas, list):
|
||||
self.db.insert_ideas(draft_name, ideas)
|
||||
return ideas
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
pass
|
||||
|
||||
try:
|
||||
text, in_tok, out_tok = self._call_claude(prompt, max_tokens=1024)
|
||||
text = self._extract_json(text)
|
||||
ideas = json.loads(text)
|
||||
if not isinstance(ideas, list):
|
||||
ideas = [ideas]
|
||||
|
||||
self.db.cache_response(
|
||||
draft_name, phash, self.config.claude_model,
|
||||
prompt, text, in_tok, out_tok,
|
||||
)
|
||||
self.db.insert_ideas(draft_name, ideas)
|
||||
return ideas
|
||||
except (json.JSONDecodeError, anthropic.APIError) as e:
|
||||
console.print(f"[red]Failed ideas for {draft_name}: {e}[/]")
|
||||
return None
|
||||
|
||||
def extract_ideas_batch(self, draft_names: list[str], cheap: bool = True) -> int:
|
||||
"""Extract ideas from multiple drafts in a single API call.
|
||||
|
||||
Uses batching to share prompt overhead — ~5x fewer API calls,
|
||||
~3x fewer tokens than individual extraction.
|
||||
"""
|
||||
drafts = []
|
||||
for name in draft_names:
|
||||
d = self.db.get_draft(name)
|
||||
if d:
|
||||
drafts.append(d)
|
||||
|
||||
if not drafts:
|
||||
return 0
|
||||
|
||||
# Build compact batch block — abstract only (no full text for batch)
|
||||
drafts_block = ""
|
||||
for d in drafts:
|
||||
drafts_block += f"\n---\n{d.name} | {d.title}\nAbstract: {d.abstract[:800]}\n"
|
||||
|
||||
prompt = BATCH_IDEAS_PROMPT.format(drafts_block=drafts_block)
|
||||
phash = _prompt_hash(prompt)
|
||||
|
||||
try:
|
||||
text, in_tok, out_tok = self._call_claude(
|
||||
prompt, max_tokens=400 * len(drafts), cheap=cheap
|
||||
)
|
||||
text = self._extract_json(text)
|
||||
results = json.loads(text)
|
||||
|
||||
if not isinstance(results, dict):
|
||||
# Fallback: if it returned a list, try to match by order
|
||||
if isinstance(results, list) and len(results) == len(drafts):
|
||||
results = {d.name: r for d, r in zip(drafts, results)}
|
||||
else:
|
||||
return 0
|
||||
|
||||
count = 0
|
||||
for d in drafts:
|
||||
ideas = results.get(d.name, [])
|
||||
if ideas:
|
||||
if not isinstance(ideas, list):
|
||||
ideas = [ideas]
|
||||
self.db.cache_response(
|
||||
d.name, _prompt_hash(f"batch-ideas-{phash}-{d.name}"),
|
||||
self.config.claude_model_cheap if cheap else self.config.claude_model,
|
||||
f"batch-ideas[{d.name}]", json.dumps(ideas),
|
||||
in_tok // len(drafts), out_tok // len(drafts),
|
||||
)
|
||||
self.db.insert_ideas(d.name, ideas)
|
||||
count += 1
|
||||
return count
|
||||
except (json.JSONDecodeError, anthropic.APIError) as e:
|
||||
console.print(f"[red]Batch ideas failed: {e}[/]")
|
||||
return 0
|
||||
|
||||
def extract_all_ideas(self, limit: int = 300, batch_size: int = 5, cheap: bool = True) -> int:
|
||||
"""Extract ideas from all drafts that don't have them yet.
|
||||
|
||||
Args:
|
||||
batch_size: Number of drafts per API call (default 5).
|
||||
Set to 1 to use individual calls with full text.
|
||||
cheap: Use Haiku model for ~10x lower cost (default True).
|
||||
"""
|
||||
missing = self.db.drafts_without_ideas(limit=limit)
|
||||
if not missing:
|
||||
console.print("All drafts already have extracted ideas.")
|
||||
return 0
|
||||
|
||||
model_label = "Haiku" if cheap else "Sonnet"
|
||||
if batch_size > 1:
|
||||
console.print(
|
||||
f"Extracting ideas from [bold]{len(missing)}[/] drafts "
|
||||
f"(batches of {batch_size}, {model_label})..."
|
||||
)
|
||||
else:
|
||||
console.print(f"Extracting ideas from [bold]{len(missing)}[/] drafts ({model_label})...")
|
||||
|
||||
count = 0
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
MofNCompleteColumn(),
|
||||
console=console,
|
||||
) as progress:
|
||||
task = progress.add_task("Extracting ideas...", total=len(missing))
|
||||
|
||||
if batch_size > 1:
|
||||
for i in range(0, len(missing), batch_size):
|
||||
batch = missing[i:i + batch_size]
|
||||
names = ", ".join(n.split("-")[-1][:10] for n in batch)
|
||||
progress.update(task, description=f"Batch: {names}")
|
||||
n = self.extract_ideas_batch(batch, cheap=cheap)
|
||||
count += n
|
||||
progress.advance(task, advance=len(batch))
|
||||
else:
|
||||
for name in missing:
|
||||
progress.update(task, description=f"Ideas: {name.split('-')[-1][:15]}")
|
||||
result = self.extract_ideas(name)
|
||||
if result:
|
||||
count += 1
|
||||
progress.advance(task)
|
||||
|
||||
in_tok, out_tok = self.db.total_tokens_used()
|
||||
console.print(
|
||||
f"Extracted ideas from [bold green]{count}[/] drafts "
|
||||
f"({self.db.idea_count()} total ideas) "
|
||||
f"| Tokens: {in_tok:,} in + {out_tok:,} out"
|
||||
)
|
||||
return count
|
||||
|
||||
def gap_analysis(self) -> list[dict]:
|
||||
"""Analyze the full landscape and identify gaps."""
|
||||
# Build compressed landscape summary
|
||||
pairs = self.db.drafts_with_ratings(limit=500)
|
||||
total = self.db.count_drafts()
|
||||
|
||||
# Category summary
|
||||
from collections import defaultdict
|
||||
cat_counts: dict[str, int] = defaultdict(int)
|
||||
for _, rating in pairs:
|
||||
for c in rating.categories:
|
||||
cat_counts[c] += 1
|
||||
category_summary = "\n".join(f"- {c}: {n} drafts" for c, n in
|
||||
sorted(cat_counts.items(), key=lambda x: x[1], reverse=True))
|
||||
|
||||
# Top ideas (if available)
|
||||
all_ideas = self.db.all_ideas()
|
||||
idea_freq: dict[str, int] = defaultdict(int)
|
||||
for idea in all_ideas:
|
||||
idea_freq[idea["title"]] += 1
|
||||
top_ideas_list = sorted(idea_freq.items(), key=lambda x: x[1], reverse=True)[:20]
|
||||
if top_ideas_list:
|
||||
top_ideas = "\n".join(f"- {title} ({count} drafts)" for title, count in top_ideas_list)
|
||||
else:
|
||||
top_ideas = "(No idea extraction data available yet)"
|
||||
|
||||
# Overlap summary — use clusters report if it exists
|
||||
overlap_summary = "Multiple clusters of near-duplicate drafts exist, particularly in:\n"
|
||||
for c, n in sorted(cat_counts.items(), key=lambda x: x[1], reverse=True)[:5]:
|
||||
overlap_summary += f"- {c} ({n} drafts, high internal overlap)\n"
|
||||
|
||||
prompt = GAP_ANALYSIS_PROMPT.format(
|
||||
total=total,
|
||||
category_summary=category_summary,
|
||||
top_ideas=top_ideas,
|
||||
overlap_summary=overlap_summary,
|
||||
)
|
||||
phash = _prompt_hash(prompt)
|
||||
|
||||
# Check cache
|
||||
cached = self.db.get_cached_response("_landscape_", phash)
|
||||
if cached:
|
||||
try:
|
||||
gaps = json.loads(cached)
|
||||
if isinstance(gaps, list):
|
||||
self.db.insert_gaps(gaps)
|
||||
return gaps
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
pass
|
||||
|
||||
try:
|
||||
text, in_tok, out_tok = self._call_claude(prompt, max_tokens=4096)
|
||||
text = self._extract_json(text)
|
||||
gaps = json.loads(text)
|
||||
if not isinstance(gaps, list):
|
||||
gaps = [gaps]
|
||||
|
||||
self.db.cache_response(
|
||||
"_landscape_", phash, self.config.claude_model,
|
||||
prompt, text, in_tok, out_tok,
|
||||
)
|
||||
self.db.insert_gaps(gaps)
|
||||
return gaps
|
||||
except (json.JSONDecodeError, anthropic.APIError) as e:
|
||||
console.print(f"[red]Gap analysis failed: {e}[/]")
|
||||
return []
|
||||
|
||||
def compare_drafts(self, draft_names: list[str]) -> str:
|
||||
"""Compare multiple drafts and return analysis text."""
|
||||
parts = []
|
||||
|
||||
Reference in New Issue
Block a user