Add auto-heal pipeline command and fix multi-source draft processing
- Add `ietf auto` command: fetches, analyzes, embeds, extracts ideas, and refreshes gaps across all sources with cost-based auto-approval - Fix SourceDocument→Draft conversion in auto fetch step - Fix gap_analysis method name in auto command - Process all 270 unrated ETSI/ISO/ITU/NIST drafts (761 total, all rated) - Update web UI templates and data layer for multi-source support Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -63,6 +63,20 @@
|
||||
<div id="treemapPlot" style="height: 450px;"></div>
|
||||
</div>
|
||||
|
||||
<!-- Cluster relationship network -->
|
||||
<div id="networkSection" class="bg-slate-900 rounded-xl border border-slate-800 p-5 mb-6 hidden">
|
||||
<h2 class="text-sm font-semibold text-slate-300 mb-1">Cluster Relationships</h2>
|
||||
<p class="text-xs text-slate-500 mb-3">Network showing how idea clusters relate to each other. Thicker lines = stronger semantic similarity. Click a link to see the connecting ideas.</p>
|
||||
<div id="networkPlot" style="height: 560px;"></div>
|
||||
<div id="linkDetail" class="hidden mt-4 bg-slate-800/50 rounded-lg p-4 border border-slate-700/50">
|
||||
<div class="flex items-center justify-between mb-2">
|
||||
<h3 class="text-sm font-semibold text-white" id="linkTitle"></h3>
|
||||
<button onclick="document.getElementById('linkDetail').classList.add('hidden')" class="text-slate-500 hover:text-white text-xs">✕</button>
|
||||
</div>
|
||||
<div id="linkContent" class="text-xs text-slate-400 space-y-2"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Cluster cards grid -->
|
||||
<h2 class="text-lg font-semibold text-white mb-4">Cluster Details</h2>
|
||||
<div id="clusterGrid" class="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4 mb-6">
|
||||
@@ -180,6 +194,135 @@ if (data.empty) {
|
||||
}, CFG);
|
||||
}
|
||||
|
||||
// --- Cluster Relationship Network ---
|
||||
const links = data.links || [];
|
||||
if (links.length > 0) {
|
||||
document.getElementById('networkSection').classList.remove('hidden');
|
||||
|
||||
// Build node set from clusters that have links
|
||||
const linkedIds = new Set();
|
||||
links.forEach(l => { linkedIds.add(l.source); linkedIds.add(l.target); });
|
||||
|
||||
const nodes = data.clusters.filter(c => linkedIds.has(c.id));
|
||||
const nodeMap = {};
|
||||
nodes.forEach((n, i) => { nodeMap[n.id] = i; });
|
||||
|
||||
// Force-directed layout using Plotly scatter + annotations for edges
|
||||
// Position nodes in a circle, then use link structure
|
||||
const n = nodes.length;
|
||||
const nodeX = nodes.map((_, i) => Math.cos(2 * Math.PI * i / n) * 4);
|
||||
const nodeY = nodes.map((_, i) => Math.sin(2 * Math.PI * i / n) * 4);
|
||||
|
||||
// Simple force-directed: pull linked nodes closer
|
||||
for (let iter = 0; iter < 80; iter++) {
|
||||
for (const link of links) {
|
||||
const si = nodeMap[link.source];
|
||||
const ti = nodeMap[link.target];
|
||||
if (si === undefined || ti === undefined) continue;
|
||||
const dx = nodeX[ti] - nodeX[si];
|
||||
const dy = nodeY[ti] - nodeY[si];
|
||||
const dist = Math.sqrt(dx*dx + dy*dy) || 1;
|
||||
const force = (link.best_pair_sim - 0.5) * 0.15;
|
||||
nodeX[si] += dx/dist * force;
|
||||
nodeY[si] += dy/dist * force;
|
||||
nodeX[ti] -= dx/dist * force;
|
||||
nodeY[ti] -= dy/dist * force;
|
||||
}
|
||||
// Repulsion between all nodes
|
||||
for (let i = 0; i < n; i++) {
|
||||
for (let j = i+1; j < n; j++) {
|
||||
const dx = nodeX[j] - nodeX[i];
|
||||
const dy = nodeY[j] - nodeY[i];
|
||||
const dist = Math.sqrt(dx*dx + dy*dy) || 0.1;
|
||||
if (dist < 1.5) {
|
||||
const repel = 0.3 / (dist * dist);
|
||||
nodeX[i] -= dx/dist * repel;
|
||||
nodeY[i] -= dy/dist * repel;
|
||||
nodeX[j] += dx/dist * repel;
|
||||
nodeY[j] += dy/dist * repel;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Edge traces (one per link for click handling)
|
||||
const edgeTraces = links.map((link, li) => {
|
||||
const si = nodeMap[link.source];
|
||||
const ti = nodeMap[link.target];
|
||||
if (si === undefined || ti === undefined) return null;
|
||||
const width = 1 + (link.best_pair_sim - 0.5) * 8;
|
||||
const opacity = 0.3 + (link.best_pair_sim - 0.5) * 1.2;
|
||||
return {
|
||||
x: [nodeX[si], nodeX[ti], null],
|
||||
y: [nodeY[si], nodeY[ti], null],
|
||||
mode: 'lines',
|
||||
line: { width: width, color: `rgba(100,116,139,${opacity})` },
|
||||
hoverinfo: 'text',
|
||||
text: `${link.source_theme} ↔ ${link.target_theme}<br>Similarity: ${(link.best_pair_sim * 100).toFixed(0)}%`,
|
||||
customdata: [li, li, null],
|
||||
showlegend: false,
|
||||
};
|
||||
}).filter(Boolean);
|
||||
|
||||
// Node trace
|
||||
const nodeTrace = {
|
||||
x: nodeX, y: nodeY,
|
||||
mode: 'markers+text',
|
||||
type: 'scatter',
|
||||
marker: {
|
||||
size: nodes.map(n => 12 + Math.sqrt(n.size) * 3),
|
||||
color: nodes.map((_, i) => PALETTE[nodes[i].id % PALETTE.length]),
|
||||
line: { width: 2, color: 'rgba(15,23,42,0.8)' },
|
||||
},
|
||||
text: nodes.map(n => n.theme.length > 25 ? n.theme.substring(0, 22) + '...' : n.theme),
|
||||
textposition: 'top center',
|
||||
textfont: { size: 10, color: '#cbd5e1' },
|
||||
hovertext: nodes.map(n =>
|
||||
`<b>${n.theme}</b><br>${n.size} ideas, ${n.drafts.length} drafts`
|
||||
),
|
||||
hoverinfo: 'text',
|
||||
showlegend: false,
|
||||
};
|
||||
|
||||
Plotly.newPlot('networkPlot', [...edgeTraces, nodeTrace], {
|
||||
...PLOTLY_LAYOUT,
|
||||
xaxis: { visible: false, showgrid: false, zeroline: false },
|
||||
yaxis: { visible: false, showgrid: false, zeroline: false },
|
||||
hovermode: 'closest',
|
||||
margin: { t: 10, r: 20, b: 10, l: 20 },
|
||||
}, CFG);
|
||||
|
||||
// Click handler for edges — show link detail
|
||||
document.getElementById('networkPlot').on('plotly_click', function(ev) {
|
||||
const pt = ev.points[0];
|
||||
if (pt.data.customdata && pt.data.customdata[pt.pointNumber] !== null) {
|
||||
const link = links[pt.data.customdata[pt.pointNumber]];
|
||||
if (!link) return;
|
||||
const detail = document.getElementById('linkDetail');
|
||||
const simPct = (link.best_pair_sim * 100).toFixed(0);
|
||||
document.getElementById('linkTitle').innerHTML =
|
||||
`<span style="color:${PALETTE[link.source % PALETTE.length]}">${link.source_theme}</span>` +
|
||||
` <span class="text-slate-500">↔</span> ` +
|
||||
`<span style="color:${PALETTE[link.target % PALETTE.length]}">${link.target_theme}</span>` +
|
||||
` <span class="text-slate-500 text-xs font-normal ml-2">${simPct}% similar</span>`;
|
||||
document.getElementById('linkContent').innerHTML = `
|
||||
<div class="grid grid-cols-2 gap-4">
|
||||
<div class="bg-slate-900/50 rounded p-3 border border-slate-700/30">
|
||||
<div class="text-slate-300 font-medium mb-1">${link.idea_a}</div>
|
||||
<a href="/drafts/${link.idea_a_draft}" class="text-blue-400/70 hover:text-blue-300 text-[10px] font-mono">${link.idea_a_draft}</a>
|
||||
</div>
|
||||
<div class="bg-slate-900/50 rounded p-3 border border-slate-700/30">
|
||||
<div class="text-slate-300 font-medium mb-1">${link.idea_b}</div>
|
||||
<a href="/drafts/${link.idea_b_draft}" class="text-blue-400/70 hover:text-blue-300 text-[10px] font-mono">${link.idea_b_draft}</a>
|
||||
</div>
|
||||
</div>
|
||||
<p class="text-slate-500 text-[10px] mt-1">These two ideas from different clusters have the strongest cross-cluster similarity.</p>
|
||||
`;
|
||||
detail.classList.remove('hidden');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// --- Cluster Cards ---
|
||||
const grid = document.getElementById('clusterGrid');
|
||||
|
||||
@@ -190,15 +333,42 @@ if (data.empty) {
|
||||
if (filter === 'large' && cluster.size < 10) return;
|
||||
|
||||
const color = PALETTE[i % PALETTE.length];
|
||||
const topIdeas = cluster.ideas.slice(0, 5);
|
||||
const ideaListHtml = topIdeas.map(idea =>
|
||||
`<li class="text-xs text-slate-400 truncate" title="${idea.description || idea.title}">
|
||||
<span class="text-slate-300">${idea.title}</span>
|
||||
</li>`
|
||||
).join('');
|
||||
const extraCount = cluster.size - topIdeas.length;
|
||||
const extraHtml = extraCount > 0
|
||||
? `<li class="text-xs text-slate-600">+${extraCount} more</li>` : '';
|
||||
const cardId = `cluster-${i}`;
|
||||
const topIdeas = cluster.ideas.slice(0, 3);
|
||||
|
||||
// Deduplicate ideas by title, track which drafts have each
|
||||
const ideaByTitle = {};
|
||||
cluster.ideas.forEach(idea => {
|
||||
if (!ideaByTitle[idea.title]) {
|
||||
ideaByTitle[idea.title] = { ...idea, drafts: [] };
|
||||
}
|
||||
ideaByTitle[idea.title].drafts.push(idea.draft_name);
|
||||
});
|
||||
const uniqueIdeas = Object.values(ideaByTitle);
|
||||
|
||||
// Preview: first 3 unique ideas
|
||||
const previewHtml = uniqueIdeas.slice(0, 3).map(idea => {
|
||||
const draftTag = idea.drafts.length > 1
|
||||
? `<span class="text-slate-600">(${idea.drafts.length} drafts)</span>`
|
||||
: `<span class="text-slate-600">${idea.drafts[0].replace('draft-', '').substring(0, 20)}</span>`;
|
||||
return `<li class="text-xs text-slate-400 truncate" title="${idea.description || idea.title}">
|
||||
<span class="text-slate-300">${idea.title}</span> ${draftTag}
|
||||
</li>`;
|
||||
}).join('');
|
||||
const previewExtra = uniqueIdeas.length > 3
|
||||
? `<li class="text-xs text-slate-600">+${uniqueIdeas.length - 3} more unique ideas</li>` : '';
|
||||
|
||||
// Full idea list (shown on expand)
|
||||
const fullIdeasHtml = uniqueIdeas.map(idea => {
|
||||
const draftLinks = idea.drafts.map(d =>
|
||||
`<a href="/drafts/${d}" class="text-blue-400/70 hover:text-blue-300 transition">${d.replace('draft-', '').substring(0, 28)}</a>`
|
||||
).join(', ');
|
||||
return `<div class="py-2 border-b border-slate-800/50 last:border-0">
|
||||
<div class="text-xs text-slate-200 font-medium">${idea.title}</div>
|
||||
${idea.description ? `<div class="text-xs text-slate-500 mt-0.5 leading-relaxed">${idea.description.substring(0, 200)}</div>` : ''}
|
||||
<div class="text-[10px] text-slate-600 mt-1 font-mono">${draftLinks}</div>
|
||||
</div>`;
|
||||
}).join('');
|
||||
|
||||
// WG badges
|
||||
const wgBadges = (cluster.wgs || []).filter(w => w.wg !== 'none').map(w =>
|
||||
@@ -224,22 +394,39 @@ if (data.empty) {
|
||||
? `<span class="text-xs bg-amber-900/30 text-amber-400 px-1.5 py-0.5 rounded">cross-WG</span>` : '';
|
||||
|
||||
const card = document.createElement('div');
|
||||
card.className = 'bg-slate-900 rounded-xl border p-5 ' +
|
||||
card.className = 'bg-slate-900 rounded-xl border p-5 cursor-pointer hover:border-slate-600 transition ' +
|
||||
(cluster.cross_wg ? 'border-amber-800/40' : 'border-slate-800');
|
||||
card.onclick = () => {
|
||||
const detail = document.getElementById(cardId);
|
||||
const chevron = document.getElementById(`chevron-${i}`);
|
||||
if (detail.classList.contains('hidden')) {
|
||||
detail.classList.remove('hidden');
|
||||
chevron.style.transform = 'rotate(180deg)';
|
||||
} else {
|
||||
detail.classList.add('hidden');
|
||||
chevron.style.transform = '';
|
||||
}
|
||||
};
|
||||
card.innerHTML = `
|
||||
<div class="flex items-center gap-2 mb-3">
|
||||
<div class="w-3 h-3 rounded-full flex-shrink-0" style="background: ${color}"></div>
|
||||
<h3 class="text-sm font-semibold text-white truncate">${cluster.theme}</h3>
|
||||
${crossBadge}
|
||||
<span class="ml-auto text-xs text-slate-500 flex-shrink-0">${cluster.size} ideas</span>
|
||||
<svg id="chevron-${i}" class="w-4 h-4 text-slate-500 flex-shrink-0 transition-transform" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"/></svg>
|
||||
</div>
|
||||
<ul class="space-y-1 mb-3">${ideaListHtml}${extraHtml}</ul>
|
||||
<ul class="space-y-1 mb-3">${previewHtml}${previewExtra}</ul>
|
||||
${(wgBadges || noneHtml) ? `<div class="mb-2"><p class="text-xs text-slate-500 mb-1">Working Groups</p><div class="flex flex-wrap gap-1">${wgBadges} ${noneHtml}</div></div>` : ''}
|
||||
${catBadges ? `<div class="mb-2"><p class="text-xs text-slate-500 mb-1">Categories</p><div class="flex flex-wrap gap-1">${catBadges}</div></div>` : ''}
|
||||
<div class="border-t border-slate-800 pt-3">
|
||||
<p class="text-xs text-slate-500 mb-1">${cluster.drafts.length} source draft${cluster.drafts.length !== 1 ? 's' : ''}</p>
|
||||
<div class="flex flex-wrap gap-1">${draftBadges}${extraDrafts}</div>
|
||||
</div>
|
||||
<!-- Expanded detail (hidden by default) -->
|
||||
<div id="${cardId}" class="hidden mt-4 border-t border-slate-700 pt-4">
|
||||
<h4 class="text-xs font-semibold text-slate-300 mb-2 uppercase tracking-wide">All ${uniqueIdeas.length} unique ideas</h4>
|
||||
<div class="max-h-80 overflow-y-auto pr-1">${fullIdeasHtml}</div>
|
||||
</div>
|
||||
`;
|
||||
grid.appendChild(card);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user