Add auto-heal pipeline command and fix multi-source draft processing

- Add `ietf auto` command: fetches, analyzes, embeds, extracts ideas,
  and refreshes gaps across all sources with cost-based auto-approval
- Fix SourceDocument→Draft conversion in auto fetch step
- Fix gap_analysis method name in auto command
- Process all 270 unrated ETSI/ISO/ITU/NIST drafts (761 total, all rated)
- Update web UI templates and data layer for multi-source support

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-08 18:41:42 +01:00
parent 1ec1f69bee
commit a46a01bd8c
15 changed files with 991 additions and 381 deletions

View File

@@ -160,29 +160,34 @@ Return ONLY a JSON object like {{"draft-name":[...], ...}}, no fences."""
# independent gap analyses and intersect results, have domain experts validate.
# ============================================================================
GAP_ANALYSIS_PROMPT = """\
You are analyzing the landscape of {total} IETF Internet-Drafts related to AI agents and autonomous systems.
You are analyzing the landscape of {total} documents related to AI agents and autonomous systems from multiple standards organizations.
## Categories and Draft Counts
## IETF Drafts — Categories and Draft Counts
{category_summary}
## Most Common Technical Ideas
## Most Common Technical Ideas (from IETF drafts)
{top_ideas}
## Known Overlap Clusters (groups of highly similar drafts)
{overlap_summary}
Identify 8-15 GAPS — areas, problems, or technical challenges NOT adequately addressed by existing drafts.
## Other Standards Bodies
{other_sources_summary}
Identify 8-15 GAPS — areas, problems, or technical challenges NOT adequately addressed by existing drafts and standards.
Return a JSON array:
[{{"topic":"short topic name","description":"2-3 sentence description","category":"closest category or new","severity":"critical|high|medium|low","evidence":"what suggests this gap matters"}}]
[{{"topic":"short topic name","description":"2-3 sentence description","category":"closest category or new","severity":"critical|high|medium|low","evidence":"what suggests this gap matters","addressed_by":"which existing standards (if any) partially address this, from any source"}}]
Focus on:
1. Problems mentioned but not solved
2. Missing infrastructure pieces
1. Problems mentioned but not solved — even across organizations
2. Missing infrastructure pieces (no standard from ANY body covers it)
3. Security/privacy/safety issues not addressed
4. Interoperability gaps between competing proposals
4. Interoperability gaps between competing proposals or between standards bodies
5. Real-world deployment concerns ignored
6. Cross-organization coordination gaps (e.g., IETF protocol needs ISO governance framework)
Consider what NIST, ISO, ETSI, ITU-T, and W3C already cover vs what remains missing.
JSON array only, no fences."""
SCORE_NOVELTY_PROMPT = """\
@@ -638,11 +643,31 @@ class Analyzer:
for c, n in sorted(cat_counts.items(), key=lambda x: x[1], reverse=True)[:5]:
overlap_summary += f"- {c} ({n} drafts, high internal overlap)\n"
# Build summary of non-IETF sources
other_rows = self.db.conn.execute(
"SELECT source, name, title, abstract FROM drafts WHERE source != 'ietf' ORDER BY source, name"
).fetchall()
source_groups: dict[str, list] = {}
for r in other_rows:
src = r["source"].upper()
source_groups.setdefault(src, []).append(r)
other_lines = []
for src, docs in sorted(source_groups.items()):
other_lines.append(f"\n### {src} ({len(docs)} documents)")
for d in docs[:30]: # cap per source to fit context
abstract = (d["abstract"] or "")[:150]
other_lines.append(f"- **{d['title'][:100]}**: {abstract}")
if len(docs) > 30:
other_lines.append(f" ... and {len(docs) - 30} more")
other_sources_summary = "\n".join(other_lines) if other_lines else "(No other sources available)"
prompt = GAP_ANALYSIS_PROMPT.format(
total=total,
category_summary=category_summary,
top_ideas=top_ideas,
overlap_summary=overlap_summary,
other_sources_summary=other_sources_summary,
)
phash = _prompt_hash(prompt)

View File

@@ -3081,3 +3081,255 @@ def export(export_type: str, fmt: str, output_file: str | None):
finally:
db.close()
# ── auto ─────────────────────────────────────────────────────────────────────
@main.command("auto")
@click.option("--cost-limit", default=2.0, help="Auto-approve operations under this USD amount (default: $2)")
@click.option("--yes", "-y", is_flag=True, help="Skip all confirmation prompts")
@click.option("--dry-run", is_flag=True, help="Show what would be done without doing it")
@click.option("--source", "-s", default=None, help="Limit to specific source (ietf,w3c,etsi,iso,itu)")
def auto(cost_limit: float, yes: bool, dry_run: bool, source: str | None):
"""Auto-heal: fetch, analyze, embed, extract ideas, and update gaps.
Automatically processes all unrated, unembedded, and idea-less drafts
across all sources. Uses cheap models (Haiku) for bulk operations.
Operations estimated above --cost-limit require confirmation.
Examples:
ietf auto # run full pipeline, auto-approve under $2
ietf auto --dry-run # show plan without executing
ietf auto -s iso # only process ISO drafts
ietf auto --cost-limit 5 # raise approval threshold to $5
ietf auto -y # skip all prompts (for cron)
"""
cfg = Config.load()
db = Database(cfg)
try:
_auto_heal(cfg, db, cost_limit=cost_limit, yes=yes, dry_run=dry_run, source_filter=source)
finally:
db.close()
def _estimate_cost(n_drafts: int, operation: str) -> float:
"""Estimate USD cost for an operation. Conservative estimates."""
# Haiku: ~$0.25/M input, ~$1.25/M output
# Sonnet: ~$3/M input, ~$15/M output
# Average draft abstract: ~500 tokens input, ~200 tokens output
costs = {
"analyze_cheap": n_drafts * 0.0005, # ~$0.50 per 1000 drafts (Haiku)
"analyze_quality": n_drafts * 0.005, # ~$5.00 per 1000 drafts (Sonnet)
"ideas_cheap": n_drafts * 0.001, # ~$1.00 per 1000 drafts (Haiku batch)
"ideas_quality": n_drafts * 0.008, # ~$8.00 per 1000 drafts (Sonnet)
"gaps": 0.05, # single Claude call
"embed": 0.0, # Ollama is free/local
"authors": 0.0, # Datatracker API is free
"fetch": 0.0, # Datatracker API is free
}
return costs.get(operation, 0.0)
def _auto_heal(cfg, db, cost_limit: float, yes: bool, dry_run: bool, source_filter: str | None):
"""Run the full auto-heal pipeline."""
import time as _time
from rich.panel import Panel
steps: list[dict] = []
total_cost = 0.0
# ── Step 1: Fetch new drafts from all sources ──
sources = [source_filter] if source_filter else cfg.observatory_sources
steps.append({
"name": f"Fetch new drafts from {', '.join(sources)}",
"sources": sources,
"cost": 0.0,
"action": "fetch",
})
# ── Step 2: Analyze unrated drafts ──
unrated = db.unrated_drafts(limit=10000)
if source_filter:
unrated = [d for d in unrated if (d.source or "ietf") == source_filter]
n_unrated = len(unrated)
analyze_cost = _estimate_cost(n_unrated, "analyze_cheap")
steps.append({
"name": f"Analyze {n_unrated} unrated drafts (Haiku)",
"count": n_unrated,
"cost": analyze_cost,
"action": "analyze",
})
total_cost += analyze_cost
# ── Step 3: Fetch authors ──
missing_authors = db.conn.execute(
"SELECT COUNT(*) FROM drafts WHERE name NOT IN (SELECT DISTINCT draft_name FROM draft_authors)"
).fetchone()[0]
steps.append({
"name": f"Fetch authors for {missing_authors} drafts",
"count": missing_authors,
"cost": 0.0,
"action": "authors",
})
# ── Step 4: Embed missing drafts ──
missing_embed = db.drafts_without_embeddings(limit=10000)
if source_filter:
source_names = {row[0] for row in db.conn.execute(
"SELECT name FROM drafts WHERE source = ?", (source_filter,)
).fetchall()}
missing_embed = [n for n in missing_embed if n in source_names]
n_embed = len(missing_embed)
steps.append({
"name": f"Embed {n_embed} drafts (Ollama, free)",
"count": n_embed,
"cost": 0.0,
"action": "embed",
})
# ── Step 5: Extract ideas ──
missing_ideas = db.drafts_without_ideas(limit=10000)
if source_filter:
if not source_names:
source_names = {row[0] for row in db.conn.execute(
"SELECT name FROM drafts WHERE source = ?", (source_filter,)
).fetchall()}
missing_ideas = [n for n in missing_ideas if n in source_names]
n_ideas = len(missing_ideas)
ideas_cost = _estimate_cost(n_ideas, "ideas_cheap")
steps.append({
"name": f"Extract ideas from {n_ideas} drafts (Haiku)",
"count": n_ideas,
"cost": ideas_cost,
"action": "ideas",
})
total_cost += ideas_cost
# ── Step 6: Refresh gaps ──
gap_cost = _estimate_cost(0, "gaps")
steps.append({
"name": "Refresh gap analysis",
"cost": gap_cost,
"action": "gaps",
})
total_cost += gap_cost
# ── Show plan ──
plan_lines = []
for s in steps:
count = s.get("count", 1)
if count == 0:
plan_lines.append(f" [dim]SKIP[/] {s['name']}")
else:
cost_str = f" [yellow]~${s['cost']:.2f}[/]" if s["cost"] > 0 else ""
plan_lines.append(f" [green]RUN[/] {s['name']}{cost_str}")
auto_approved = total_cost <= cost_limit
plan_lines.append(f"\n [bold]Estimated total cost: ${total_cost:.2f}[/]")
if auto_approved:
plan_lines.append(f" [green]Auto-approved (under ${cost_limit:.2f} limit)[/]")
else:
plan_lines.append(f" [yellow]Requires approval (over ${cost_limit:.2f} limit)[/]")
console.print(Panel("\n".join(plan_lines), title="Auto-Heal Plan"))
if dry_run:
console.print("[bold yellow]DRY RUN[/] — no changes made.")
return
# ── Approval ──
if not auto_approved and not yes:
if not click.confirm(f"Estimated cost ${total_cost:.2f} exceeds ${cost_limit:.2f} limit. Proceed?"):
console.print("[yellow]Aborted.[/]")
return
# ── Execute ──
start = _time.time()
for step in steps:
action = step["action"]
count = step.get("count", 0)
if action == "fetch":
console.print(f"\n[bold cyan]>>> Fetching from {step['sources']}...[/]")
from .sources import get_fetcher
from .observatory import _doc_to_draft
for src_name in step["sources"]:
try:
fetcher = get_fetcher(src_name, cfg)
before = db.count_drafts()
results = fetcher.search(keywords=cfg.search_keywords)
for doc in results:
db.upsert_draft(_doc_to_draft(doc))
after = db.count_drafts()
new = after - before
console.print(f" [{src_name}] +{new} new drafts")
fetcher.close()
except Exception as e:
console.print(f" [{src_name}] [red]Error: {e}[/]")
elif action == "analyze" and count > 0:
console.print(f"\n[bold cyan]>>> Analyzing {count} drafts (Haiku)...[/]")
from .analyzer import Analyzer
analyzer = Analyzer(cfg, db)
orig_model = cfg.claude_model
cfg.claude_model = cfg.claude_model_cheap
try:
done = analyzer.rate_all_unrated(limit=count)
console.print(f" Analyzed [bold green]{done}[/] drafts")
finally:
cfg.claude_model = orig_model
elif action == "authors" and count > 0:
console.print(f"\n[bold cyan]>>> Fetching authors for {count} drafts...[/]")
from .authors import AuthorNetwork
author_net = AuthorNetwork(cfg, db)
done = author_net.fetch_all_authors()
console.print(f" Fetched authors for [bold green]{done}[/] drafts")
elif action == "embed" and count > 0:
console.print(f"\n[bold cyan]>>> Embedding {count} drafts (Ollama)...[/]")
from .embeddings import Embedder
with Embedder(cfg, db) as embedder:
done = embedder.embed_all_missing()
console.print(f" Embedded [bold green]{done}[/] drafts")
elif action == "ideas" and count > 0:
console.print(f"\n[bold cyan]>>> Extracting ideas from {count} drafts (Haiku)...[/]")
from .analyzer import Analyzer
analyzer = Analyzer(cfg, db)
done = analyzer.extract_all_ideas(limit=count, batch_size=5, cheap=True)
console.print(f" Extracted ideas from [bold green]{done}[/] drafts")
elif action == "gaps":
console.print(f"\n[bold cyan]>>> Refreshing gap analysis...[/]")
from .analyzer import Analyzer
analyzer = Analyzer(cfg, db)
gaps = analyzer.gap_analysis()
if gaps:
console.print(f" Found [bold green]{len(gaps)}[/] gaps")
elapsed = _time.time() - start
console.print(f"\n[bold green]Auto-heal complete![/] ({elapsed:.1f}s, ~${total_cost:.2f})")
# Show final counts
total = db.count_drafts()
rated = db.conn.execute("SELECT COUNT(*) FROM ratings").fetchone()[0]
embedded = db.conn.execute("SELECT COUNT(*) FROM embeddings").fetchone()[0]
idea_count = db.conn.execute("SELECT COUNT(*) FROM ideas").fetchone()[0]
gap_count = db.conn.execute("SELECT COUNT(*) FROM gaps").fetchone()[0]
console.print(f" Drafts: {total} | Rated: {rated} | Embedded: {embedded} | Ideas: {idea_count} | Gaps: {gap_count}")
by_source = db.conn.execute(
"SELECT source, COUNT(*) FROM drafts GROUP BY source ORDER BY COUNT(*) DESC"
).fetchall()
source_str = " | ".join(f"{s}: {c}" for s, c in by_source)
console.print(f" Sources: {source_str}")

View File

@@ -52,7 +52,7 @@ class Config:
# Observatory — add "w3c" to enable W3C spec tracking:
# ietf observatory update --source w3c (one-off)
# or set observatory_sources to ["ietf", "w3c"] in config.json
observatory_sources: list[str] = field(default_factory=lambda: ["ietf", "w3c", "etsi", "itu", "iso"])
observatory_sources: list[str] = field(default_factory=lambda: ["ietf", "w3c", "etsi", "itu", "iso", "nist"])
dashboard_dir: str = str(DEFAULT_DATA_DIR.parent / "docs")
w3c_groups: list[str] = field(default_factory=lambda: [
"webmachinelearning", "wot", "credentials", "did", "vc"

View File

@@ -5,6 +5,7 @@ from .etsi import ETSIFetcher
from .ietf import IETFFetcher
from .iso import ISOFetcher
from .itu import ITUFetcher
from .nist import NISTFetcher
from .w3c import W3CFetcher
FETCHERS = {
@@ -13,6 +14,7 @@ FETCHERS = {
"etsi": ETSIFetcher,
"itu": ITUFetcher,
"iso": ISOFetcher,
"nist": NISTFetcher,
}

View File

@@ -151,10 +151,11 @@ class ISOFetcher:
continue
name = _iso_id_to_name(ref)
abstract = scope[:2000] if scope else f"ISO/IEC standard: {title}. Committee: {committee}."
docs.append(SourceDocument(
name=name,
title=f"{ref}: {title}",
abstract=f"ISO/IEC standard: {title}. Committee: {committee}. Status: {status}.",
abstract=abstract,
source="iso",
source_id=ref,
source_url=f"https://www.iso.org/standard/{ref.split(':')[0].replace('/', '%2F').replace(' ', '%20')}.html",

View File

@@ -892,11 +892,13 @@ def _compute_author_network_full(db: Database) -> AuthorNetwork:
if len(component) >= 2:
org_mix: dict[str, int] = Counter()
member_orgs: dict[str, str] = {}
cluster_drafts: dict[str, str] = {} # name -> title
for m in component:
org = author_info.get(m, {}).get("org", "")
if org:
org_mix[org] += 1
member_orgs[m] = org
for dn in author_info.get(m, {}).get("drafts", []):
if dn not in cluster_drafts:
d = _all_drafts_map.get(dn)
@@ -904,9 +906,10 @@ def _compute_author_network_full(db: Database) -> AuthorNetwork:
clusters.append({
"id": len(clusters),
"members": component,
"member_orgs": member_orgs,
"org_mix": dict(org_mix.most_common()),
"size": len(component),
"drafts": [{"name": n, "title": t} for n, t in list(cluster_drafts.items())[:15]],
"drafts": [{"name": n, "title": t} for n, t in list(cluster_drafts.items())],
"draft_count": len(cluster_drafts),
})
@@ -1062,11 +1065,78 @@ def _compute_idea_clusters(db: Database) -> dict:
except Exception:
pass
# --- Cross-cluster links ---
# Find pairs of clusters whose ideas are semantically related
# Use centroid similarity + best idea-pair links
links = []
if len(clusters) >= 2:
# Build cluster centroids from normalized embeddings
cluster_centroids = {}
cluster_member_indices: dict[int, list[int]] = defaultdict(list)
for idx, iid in enumerate(idea_ids):
cid = iid_to_new.get(iid, int(labels[idx]))
cluster_member_indices[cid].append(idx)
for cid, indices in cluster_member_indices.items():
if indices:
centroid = matrix_norm[indices].mean(axis=0)
norm = np.linalg.norm(centroid)
if norm > 0:
cluster_centroids[cid] = centroid / norm
# Compute pairwise centroid similarity for all cluster pairs
cids_sorted = sorted(cluster_centroids.keys())
for ci_idx, ci in enumerate(cids_sorted):
for cj in cids_sorted[ci_idx + 1:]:
sim = float(np.dot(cluster_centroids[ci], cluster_centroids[cj]))
if sim < 0.45:
continue
# Find the best idea pair across these two clusters
best_sim = 0.0
best_pair = (None, None)
# Sample up to 20 ideas per cluster to keep it fast
ci_members = cluster_member_indices[ci][:20]
cj_members = cluster_member_indices[cj][:20]
for mi in ci_members:
for mj in cj_members:
pair_sim = float(np.dot(matrix_norm[mi], matrix_norm[mj]))
if pair_sim > best_sim:
best_sim = pair_sim
best_pair = (idea_ids[mi], idea_ids[mj])
if best_sim < 0.5:
continue
# Get theme names
ci_theme = next((c["theme"] for c in clusters if c["id"] == ci), f"Cluster {ci}")
cj_theme = next((c["theme"] for c in clusters if c["id"] == cj), f"Cluster {cj}")
idea_a = idea_map.get(best_pair[0], {})
idea_b = idea_map.get(best_pair[1], {})
links.append({
"source": ci,
"target": cj,
"source_theme": ci_theme,
"target_theme": cj_theme,
"similarity": round(sim, 3),
"best_pair_sim": round(best_sim, 3),
"idea_a": idea_a.get("title", ""),
"idea_a_draft": idea_a.get("draft_name", ""),
"idea_b": idea_b.get("title", ""),
"idea_b_draft": idea_b.get("draft_name", ""),
})
links.sort(key=lambda l: l["best_pair_sim"], reverse=True)
links = links[:50] # cap at top 50 links
total = len(idea_ids)
clustered = sum(c["size"] for c in clusters)
return {
"clusters": clusters,
"scatter": scatter,
"links": links,
"stats": {"total": total, "clustered": clustered, "num_clusters": len(clusters)},
"empty": False,
}

View File

@@ -116,34 +116,72 @@
<p class="text-xs text-slate-500 mb-4">Clusters are formed by connected-component analysis of the co-authorship graph: authors who share 2+ drafts are linked, and all authors reachable through such links form a cluster. This reveals research teams and institutional collaboration patterns — a cluster of 20 authors from 3 organizations means those groups actively co-author across org boundaries. Click a cluster to highlight it in the graph above.</p>
<div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4" id="clusterGrid">
{% for c in network.clusters[:12] %}
<div class="cluster-card bg-slate-800/50 rounded-lg border border-slate-700/50 p-4 cursor-pointer" data-cluster-id="{{ c.id }}" onclick="highlightCluster({{ c.id }})">
<div class="flex items-center justify-between mb-2">
<div class="cluster-card bg-slate-800/50 rounded-lg border border-slate-700/50 p-4 cursor-pointer" data-cluster-id="{{ c.id }}">
<!-- Header — click to highlight in graph -->
<div class="flex items-center justify-between mb-2" onclick="highlightCluster({{ c.id }})">
<span class="text-sm font-semibold text-white">Cluster #{{ c.id + 1 }}</span>
<div class="flex gap-1.5">
<div class="flex gap-1.5 items-center">
<span class="text-xs px-2 py-0.5 rounded-full bg-blue-500/20 text-blue-400">{{ c.size }} authors</span>
<span class="text-xs px-2 py-0.5 rounded-full bg-emerald-500/20 text-emerald-400">{{ c.draft_count }} drafts</span>
<svg class="w-4 h-4 text-slate-500 transition-transform cluster-chevron" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"/></svg>
</div>
</div>
<!-- Org mix -->
<div class="flex flex-wrap gap-1 mb-2">
{% for org, count in c.org_mix.items() %}
<span class="text-xs px-2 py-0.5 rounded-full bg-slate-700 text-slate-300">{{ org }} ({{ count }})</span>
{% endfor %}
</div>
<!-- Preview: first 3 members -->
<div class="text-xs text-slate-500 mb-2 truncate" title="{{ c.members | join(', ') }}">
{{ c.members[:5] | join(', ') }}{% if c.members | length > 5 %} +{{ c.members | length - 5 }} more{% endif %}
{{ c.members[:3] | join(', ') }}{% if c.members | length > 3 %} +{{ c.members | length - 3 }} more{% endif %}
</div>
<!-- Preview: first 3 drafts -->
{% if c.drafts %}
<div class="border-t border-slate-700/50 pt-2 mt-2">
{% for d in c.drafts[:5] %}
{% for d in c.drafts[:3] %}
<div class="text-xs truncate mb-0.5" title="{{ d.name }}: {{ d.title }}">
<a href="/drafts/{{ d.name }}" class="text-blue-400/70 hover:text-blue-300 transition" onclick="event.stopPropagation()">{{ d.title }}</a>
</div>
{% endfor %}
{% if c.draft_count > 5 %}
<div class="text-xs text-slate-600 mt-1">+{{ c.draft_count - 5 }} more drafts</div>
{% if c.draft_count > 3 %}
<div class="text-xs text-slate-600 mt-1">+{{ c.draft_count - 3 }} more drafts</div>
{% endif %}
</div>
{% endif %}
<!-- Expanded detail (hidden by default) -->
<div class="cluster-detail hidden mt-3 border-t border-slate-700/50 pt-3" id="authorCluster-{{ c.id }}">
<!-- All members with org -->
<h4 class="text-xs font-semibold text-slate-300 mb-2 uppercase tracking-wide">All {{ c.size }} Authors</h4>
<div class="max-h-48 overflow-y-auto mb-3 space-y-1">
{% for member in c.members %}
<div class="text-xs flex items-center justify-between gap-2">
<a href="/drafts?q={{ member | urlencode }}" class="text-slate-300 hover:text-blue-400 transition truncate" onclick="event.stopPropagation()">{{ member }}</a>
{% set member_org = c.member_orgs[member] if c.member_orgs is defined and member in c.member_orgs else '' %}
{% if member_org %}
<span class="text-slate-600 text-[10px] truncate max-w-[120px] flex-shrink-0">{{ member_org }}</span>
{% endif %}
</div>
{% endfor %}
</div>
<!-- All drafts -->
{% if c.drafts %}
<h4 class="text-xs font-semibold text-slate-300 mb-2 uppercase tracking-wide">All {{ c.draft_count }} Drafts</h4>
<div class="max-h-48 overflow-y-auto space-y-1.5">
{% for d in c.drafts %}
<div class="text-xs" title="{{ d.name }}">
<a href="/drafts/{{ d.name }}" class="text-blue-400/70 hover:text-blue-300 transition" onclick="event.stopPropagation()">{{ d.title }}</a>
<span class="text-slate-600 font-mono text-[10px] ml-1">{{ d.name | replace('draft-', '') | truncate(25) }}</span>
</div>
{% endfor %}
</div>
{% endif %}
</div>
</div>
{% endfor %}
</div>
@@ -478,6 +516,20 @@ const network = {{ network | tojson }};
});
});
// Toggle expand/collapse on cluster card chevron click
document.querySelectorAll('.cluster-card').forEach(card => {
card.addEventListener('click', function(e) {
// Don't toggle if clicking a link or the highlight header
if (e.target.closest('a') || e.target.closest('[onclick*="highlightCluster"]')) return;
const detail = card.querySelector('.cluster-detail');
const chevron = card.querySelector('.cluster-chevron');
if (detail) {
detail.classList.toggle('hidden');
chevron.style.transform = detail.classList.contains('hidden') ? '' : 'rotate(180deg)';
}
});
});
// Expose cluster highlighting globally
window.highlightCluster = function(clusterId) {
const cluster = (network.clusters || []).find(c => c.id === clusterId);

View File

@@ -71,7 +71,7 @@
{{ draft.title }}
</a>
<div class="text-xs text-slate-600 font-mono mt-1">{{ draft.name }}</div>
<div class="text-xs text-slate-500 mt-2 line-clamp-3">{{ draft.abstract[:200] }}</div>
<div class="text-xs text-slate-500 mt-2 line-clamp-3">{{ (draft.abstract | striptags)[:200] }}</div>
{% if draft.rating %}
<!-- Rating radar -->
@@ -79,9 +79,15 @@
{% for dim, label in [('novelty', 'Nov'), ('maturity', 'Mat'), ('relevance', 'Rel'), ('momentum', 'Mom'), ('overlap', 'Ovl')] %}
<div>
<div class="text-xs text-slate-500">{{ label }}</div>
{% if dim == 'overlap' %}
<div class="text-sm font-semibold {% if draft.rating[dim] <= 2 %}text-green-400{% elif draft.rating[dim] <= 3 %}text-yellow-400{% else %}text-red-400{% endif %}">
{{ draft.rating[dim] }}
</div>
{% else %}
<div class="text-sm font-semibold {% if draft.rating[dim] >= 4 %}text-green-400{% elif draft.rating[dim] >= 3 %}text-yellow-400{% else %}text-red-400{% endif %}">
{{ draft.rating[dim] }}
</div>
{% endif %}
</div>
{% endfor %}
</div>

View File

@@ -91,7 +91,7 @@
<svg class="w-4 h-4 text-slate-500" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 6h16M4 12h16M4 18h7"/></svg>
Abstract
</h2>
<p class="text-sm text-slate-400 leading-relaxed">{{ draft.abstract or "No abstract available." }}</p>
<p class="text-sm text-slate-400 leading-relaxed">{{ (draft.abstract | striptags) or "No abstract available." }}</p>
</div>
<!-- Rating Analysis -->
@@ -120,10 +120,18 @@
<svg class="w-3.5 h-3.5 text-slate-500" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="{{ icon }}"/></svg>
<span class="text-xs font-semibold text-slate-300 uppercase tracking-wide">{{ label }}</span>
</div>
{% if dim == "overlap" %}
<span class="text-lg font-bold {% if val <= 2 %}text-green-400{% elif val <= 3 %}text-amber-400{% else %}text-red-400{% endif %}">{{ val }}<span class="text-xs text-slate-600 font-normal">/5</span></span>
{% else %}
<span class="text-lg font-bold {% if val >= 4 %}text-green-400{% elif val >= 3 %}text-amber-400{% else %}text-red-400{% endif %}">{{ val }}<span class="text-xs text-slate-600 font-normal">/5</span></span>
{% endif %}
</div>
<div class="dim-progress mb-2">
{% if dim == "overlap" %}
<div class="dim-progress-fill {% if val <= 2 %}dim-high{% elif val <= 3 %}dim-mid{% else %}dim-low{% endif %}" style="width: {{ val * 20 }}%"></div>
{% else %}
<div class="dim-progress-fill {% if val >= 4 %}dim-high{% elif val >= 3 %}dim-mid{% else %}dim-low{% endif %}" style="width: {{ val * 20 }}%"></div>
{% endif %}
</div>
{% if draft.rating[dim + '_note'] %}
<p class="text-xs text-slate-500 leading-relaxed">{{ draft.rating[dim + '_note'] }}</p>
@@ -231,7 +239,11 @@
{% for dim, abbr in [("novelty","N"), ("maturity","M"), ("overlap","O"), ("momentum","Mo"), ("relevance","R")] %}
{% set v = draft.rating[dim] %}
<div>
{% if dim == "overlap" %}
<div class="text-xs font-bold {% if v <= 2 %}text-green-400{% elif v <= 3 %}text-amber-400{% else %}text-red-400{% endif %}">{{ v }}</div>
{% else %}
<div class="text-xs font-bold {% if v >= 4 %}text-green-400{% elif v >= 3 %}text-amber-400{% else %}text-red-400{% endif %}">{{ v }}</div>
{% endif %}
<div class="text-[9px] text-slate-600 uppercase">{{ abbr }}</div>
</div>
{% endfor %}

View File

@@ -68,6 +68,11 @@
color: #c084fc;
border: 1px solid rgba(168, 85, 247, 0.3);
}
.source-nist {
background: rgba(6, 182, 212, 0.15);
color: #22d3ee;
border: 1px solid rgba(6, 182, 212, 0.3);
}
.source-generated {
background: rgba(148, 163, 184, 0.15);
color: #94a3b8;
@@ -180,6 +185,7 @@
<option value="etsi" {% if current_source == 'etsi' %}selected{% endif %}>ETSI</option>
<option value="itu" {% if current_source == 'itu' %}selected{% endif %}>ITU-T</option>
<option value="iso" {% if current_source == 'iso' %}selected{% endif %}>ISO/IEC</option>
<option value="nist" {% if current_source == 'nist' %}selected{% endif %}>NIST</option>
</select>
</div>
<!-- Sort -->
@@ -426,7 +432,7 @@
<td class="px-4 py-3 hidden xl:table-cell">
<div class="flex items-center gap-1.5">
<span class="dim-bar-bg">
<span class="dim-bar-fill {% if d.overlap >= 4 %}dim-fill-high{% elif d.overlap >= 3 %}dim-fill-mid{% else %}dim-fill-low{% endif %}"
<span class="dim-bar-fill {% if d.overlap <= 2 %}dim-fill-high{% elif d.overlap <= 3 %}dim-fill-mid{% else %}dim-fill-low{% endif %}"
style="width: {{ (d.overlap / 5 * 100)|int }}%"></span>
</span>
<span class="text-xs text-slate-500 font-mono w-4 text-right">{{ d.overlap }}</span>

View File

@@ -63,6 +63,20 @@
<div id="treemapPlot" style="height: 450px;"></div>
</div>
<!-- Cluster relationship network -->
<div id="networkSection" class="bg-slate-900 rounded-xl border border-slate-800 p-5 mb-6 hidden">
<h2 class="text-sm font-semibold text-slate-300 mb-1">Cluster Relationships</h2>
<p class="text-xs text-slate-500 mb-3">Network showing how idea clusters relate to each other. Thicker lines = stronger semantic similarity. Click a link to see the connecting ideas.</p>
<div id="networkPlot" style="height: 560px;"></div>
<div id="linkDetail" class="hidden mt-4 bg-slate-800/50 rounded-lg p-4 border border-slate-700/50">
<div class="flex items-center justify-between mb-2">
<h3 class="text-sm font-semibold text-white" id="linkTitle"></h3>
<button onclick="document.getElementById('linkDetail').classList.add('hidden')" class="text-slate-500 hover:text-white text-xs"></button>
</div>
<div id="linkContent" class="text-xs text-slate-400 space-y-2"></div>
</div>
</div>
<!-- Cluster cards grid -->
<h2 class="text-lg font-semibold text-white mb-4">Cluster Details</h2>
<div id="clusterGrid" class="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4 mb-6">
@@ -180,6 +194,135 @@ if (data.empty) {
}, CFG);
}
// --- Cluster Relationship Network ---
const links = data.links || [];
if (links.length > 0) {
document.getElementById('networkSection').classList.remove('hidden');
// Build node set from clusters that have links
const linkedIds = new Set();
links.forEach(l => { linkedIds.add(l.source); linkedIds.add(l.target); });
const nodes = data.clusters.filter(c => linkedIds.has(c.id));
const nodeMap = {};
nodes.forEach((n, i) => { nodeMap[n.id] = i; });
// Force-directed layout using Plotly scatter + annotations for edges
// Position nodes in a circle, then use link structure
const n = nodes.length;
const nodeX = nodes.map((_, i) => Math.cos(2 * Math.PI * i / n) * 4);
const nodeY = nodes.map((_, i) => Math.sin(2 * Math.PI * i / n) * 4);
// Simple force-directed: pull linked nodes closer
for (let iter = 0; iter < 80; iter++) {
for (const link of links) {
const si = nodeMap[link.source];
const ti = nodeMap[link.target];
if (si === undefined || ti === undefined) continue;
const dx = nodeX[ti] - nodeX[si];
const dy = nodeY[ti] - nodeY[si];
const dist = Math.sqrt(dx*dx + dy*dy) || 1;
const force = (link.best_pair_sim - 0.5) * 0.15;
nodeX[si] += dx/dist * force;
nodeY[si] += dy/dist * force;
nodeX[ti] -= dx/dist * force;
nodeY[ti] -= dy/dist * force;
}
// Repulsion between all nodes
for (let i = 0; i < n; i++) {
for (let j = i+1; j < n; j++) {
const dx = nodeX[j] - nodeX[i];
const dy = nodeY[j] - nodeY[i];
const dist = Math.sqrt(dx*dx + dy*dy) || 0.1;
if (dist < 1.5) {
const repel = 0.3 / (dist * dist);
nodeX[i] -= dx/dist * repel;
nodeY[i] -= dy/dist * repel;
nodeX[j] += dx/dist * repel;
nodeY[j] += dy/dist * repel;
}
}
}
}
// Edge traces (one per link for click handling)
const edgeTraces = links.map((link, li) => {
const si = nodeMap[link.source];
const ti = nodeMap[link.target];
if (si === undefined || ti === undefined) return null;
const width = 1 + (link.best_pair_sim - 0.5) * 8;
const opacity = 0.3 + (link.best_pair_sim - 0.5) * 1.2;
return {
x: [nodeX[si], nodeX[ti], null],
y: [nodeY[si], nodeY[ti], null],
mode: 'lines',
line: { width: width, color: `rgba(100,116,139,${opacity})` },
hoverinfo: 'text',
text: `${link.source_theme}${link.target_theme}<br>Similarity: ${(link.best_pair_sim * 100).toFixed(0)}%`,
customdata: [li, li, null],
showlegend: false,
};
}).filter(Boolean);
// Node trace
const nodeTrace = {
x: nodeX, y: nodeY,
mode: 'markers+text',
type: 'scatter',
marker: {
size: nodes.map(n => 12 + Math.sqrt(n.size) * 3),
color: nodes.map((_, i) => PALETTE[nodes[i].id % PALETTE.length]),
line: { width: 2, color: 'rgba(15,23,42,0.8)' },
},
text: nodes.map(n => n.theme.length > 25 ? n.theme.substring(0, 22) + '...' : n.theme),
textposition: 'top center',
textfont: { size: 10, color: '#cbd5e1' },
hovertext: nodes.map(n =>
`<b>${n.theme}</b><br>${n.size} ideas, ${n.drafts.length} drafts`
),
hoverinfo: 'text',
showlegend: false,
};
Plotly.newPlot('networkPlot', [...edgeTraces, nodeTrace], {
...PLOTLY_LAYOUT,
xaxis: { visible: false, showgrid: false, zeroline: false },
yaxis: { visible: false, showgrid: false, zeroline: false },
hovermode: 'closest',
margin: { t: 10, r: 20, b: 10, l: 20 },
}, CFG);
// Click handler for edges — show link detail
document.getElementById('networkPlot').on('plotly_click', function(ev) {
const pt = ev.points[0];
if (pt.data.customdata && pt.data.customdata[pt.pointNumber] !== null) {
const link = links[pt.data.customdata[pt.pointNumber]];
if (!link) return;
const detail = document.getElementById('linkDetail');
const simPct = (link.best_pair_sim * 100).toFixed(0);
document.getElementById('linkTitle').innerHTML =
`<span style="color:${PALETTE[link.source % PALETTE.length]}">${link.source_theme}</span>` +
` <span class="text-slate-500">↔</span> ` +
`<span style="color:${PALETTE[link.target % PALETTE.length]}">${link.target_theme}</span>` +
` <span class="text-slate-500 text-xs font-normal ml-2">${simPct}% similar</span>`;
document.getElementById('linkContent').innerHTML = `
<div class="grid grid-cols-2 gap-4">
<div class="bg-slate-900/50 rounded p-3 border border-slate-700/30">
<div class="text-slate-300 font-medium mb-1">${link.idea_a}</div>
<a href="/drafts/${link.idea_a_draft}" class="text-blue-400/70 hover:text-blue-300 text-[10px] font-mono">${link.idea_a_draft}</a>
</div>
<div class="bg-slate-900/50 rounded p-3 border border-slate-700/30">
<div class="text-slate-300 font-medium mb-1">${link.idea_b}</div>
<a href="/drafts/${link.idea_b_draft}" class="text-blue-400/70 hover:text-blue-300 text-[10px] font-mono">${link.idea_b_draft}</a>
</div>
</div>
<p class="text-slate-500 text-[10px] mt-1">These two ideas from different clusters have the strongest cross-cluster similarity.</p>
`;
detail.classList.remove('hidden');
}
});
}
// --- Cluster Cards ---
const grid = document.getElementById('clusterGrid');
@@ -190,15 +333,42 @@ if (data.empty) {
if (filter === 'large' && cluster.size < 10) return;
const color = PALETTE[i % PALETTE.length];
const topIdeas = cluster.ideas.slice(0, 5);
const ideaListHtml = topIdeas.map(idea =>
`<li class="text-xs text-slate-400 truncate" title="${idea.description || idea.title}">
<span class="text-slate-300">${idea.title}</span>
</li>`
).join('');
const extraCount = cluster.size - topIdeas.length;
const extraHtml = extraCount > 0
? `<li class="text-xs text-slate-600">+${extraCount} more</li>` : '';
const cardId = `cluster-${i}`;
const topIdeas = cluster.ideas.slice(0, 3);
// Deduplicate ideas by title, track which drafts have each
const ideaByTitle = {};
cluster.ideas.forEach(idea => {
if (!ideaByTitle[idea.title]) {
ideaByTitle[idea.title] = { ...idea, drafts: [] };
}
ideaByTitle[idea.title].drafts.push(idea.draft_name);
});
const uniqueIdeas = Object.values(ideaByTitle);
// Preview: first 3 unique ideas
const previewHtml = uniqueIdeas.slice(0, 3).map(idea => {
const draftTag = idea.drafts.length > 1
? `<span class="text-slate-600">(${idea.drafts.length} drafts)</span>`
: `<span class="text-slate-600">${idea.drafts[0].replace('draft-', '').substring(0, 20)}</span>`;
return `<li class="text-xs text-slate-400 truncate" title="${idea.description || idea.title}">
<span class="text-slate-300">${idea.title}</span> ${draftTag}
</li>`;
}).join('');
const previewExtra = uniqueIdeas.length > 3
? `<li class="text-xs text-slate-600">+${uniqueIdeas.length - 3} more unique ideas</li>` : '';
// Full idea list (shown on expand)
const fullIdeasHtml = uniqueIdeas.map(idea => {
const draftLinks = idea.drafts.map(d =>
`<a href="/drafts/${d}" class="text-blue-400/70 hover:text-blue-300 transition">${d.replace('draft-', '').substring(0, 28)}</a>`
).join(', ');
return `<div class="py-2 border-b border-slate-800/50 last:border-0">
<div class="text-xs text-slate-200 font-medium">${idea.title}</div>
${idea.description ? `<div class="text-xs text-slate-500 mt-0.5 leading-relaxed">${idea.description.substring(0, 200)}</div>` : ''}
<div class="text-[10px] text-slate-600 mt-1 font-mono">${draftLinks}</div>
</div>`;
}).join('');
// WG badges
const wgBadges = (cluster.wgs || []).filter(w => w.wg !== 'none').map(w =>
@@ -224,22 +394,39 @@ if (data.empty) {
? `<span class="text-xs bg-amber-900/30 text-amber-400 px-1.5 py-0.5 rounded">cross-WG</span>` : '';
const card = document.createElement('div');
card.className = 'bg-slate-900 rounded-xl border p-5 ' +
card.className = 'bg-slate-900 rounded-xl border p-5 cursor-pointer hover:border-slate-600 transition ' +
(cluster.cross_wg ? 'border-amber-800/40' : 'border-slate-800');
card.onclick = () => {
const detail = document.getElementById(cardId);
const chevron = document.getElementById(`chevron-${i}`);
if (detail.classList.contains('hidden')) {
detail.classList.remove('hidden');
chevron.style.transform = 'rotate(180deg)';
} else {
detail.classList.add('hidden');
chevron.style.transform = '';
}
};
card.innerHTML = `
<div class="flex items-center gap-2 mb-3">
<div class="w-3 h-3 rounded-full flex-shrink-0" style="background: ${color}"></div>
<h3 class="text-sm font-semibold text-white truncate">${cluster.theme}</h3>
${crossBadge}
<span class="ml-auto text-xs text-slate-500 flex-shrink-0">${cluster.size} ideas</span>
<svg id="chevron-${i}" class="w-4 h-4 text-slate-500 flex-shrink-0 transition-transform" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"/></svg>
</div>
<ul class="space-y-1 mb-3">${ideaListHtml}${extraHtml}</ul>
<ul class="space-y-1 mb-3">${previewHtml}${previewExtra}</ul>
${(wgBadges || noneHtml) ? `<div class="mb-2"><p class="text-xs text-slate-500 mb-1">Working Groups</p><div class="flex flex-wrap gap-1">${wgBadges} ${noneHtml}</div></div>` : ''}
${catBadges ? `<div class="mb-2"><p class="text-xs text-slate-500 mb-1">Categories</p><div class="flex flex-wrap gap-1">${catBadges}</div></div>` : ''}
<div class="border-t border-slate-800 pt-3">
<p class="text-xs text-slate-500 mb-1">${cluster.drafts.length} source draft${cluster.drafts.length !== 1 ? 's' : ''}</p>
<div class="flex flex-wrap gap-1">${draftBadges}${extraDrafts}</div>
</div>
<!-- Expanded detail (hidden by default) -->
<div id="${cardId}" class="hidden mt-4 border-t border-slate-700 pt-4">
<h4 class="text-xs font-semibold text-slate-300 mb-2 uppercase tracking-wide">All ${uniqueIdeas.length} unique ideas</h4>
<div class="max-h-80 overflow-y-auto pr-1">${fullIdeasHtml}</div>
</div>
`;
grid.appendChild(card);
});

View File

@@ -185,8 +185,13 @@ document.getElementById('scatter').on('plotly_click', function(data) {
return 'score-low';
}
function dimBadge(val) {
const cls = val >= 4 ? 'text-green-400' : val >= 3 ? 'text-yellow-400' : 'text-slate-500';
function dimBadge(val, inverted = false) {
let cls;
if (inverted) {
cls = val <= 2 ? 'text-green-400' : val <= 3 ? 'text-yellow-400' : 'text-red-400';
} else {
cls = val >= 4 ? 'text-green-400' : val >= 3 ? 'text-yellow-400' : 'text-slate-500';
}
return `<span class="${cls}">${val}</span>`;
}
@@ -207,7 +212,7 @@ document.getElementById('scatter').on('plotly_click', function(data) {
<td class="px-4 py-3 text-center">${dimBadge(d.maturity)}</td>
<td class="px-4 py-3 text-center">${dimBadge(d.relevance)}</td>
<td class="px-4 py-3 text-center">${dimBadge(d.momentum)}</td>
<td class="px-4 py-3 text-center">${dimBadge(d.overlap)}</td>
<td class="px-4 py-3 text-center">${dimBadge(d.overlap, true)}</td>
<td class="px-4 py-3">
<span class="px-2 py-0.5 rounded text-[10px] bg-slate-800 text-slate-400">${d.category}</span>
</td>