v0.3.0: Gap-to-Draft pipeline, Living Standards Observatory, blog series

Gap-to-Draft Pipeline (ietf pipeline):
- Context builder assembles ideas, RFC foundations, similar drafts, ecosystem vision
- Generator produces outlines + sections using rich context with Claude
- Quality gates: novelty (embedding similarity), references, format, self-rating
- Family coordinator generates 5-draft ecosystem (AEM/ATD/HITL/AEPB/APAE)
- I-D formatter with proper headers, references, 72-char wrapping

Living Standards Observatory (ietf observatory):
- Source abstraction with IETF + W3C fetchers
- 7-step update pipeline: snapshot, fetch, analyze, embed, ideas, gaps, record
- Static GitHub Pages dashboard (explorer, gap tracker, timeline)
- Weekly CI/CD automation via GitHub Actions

Also includes:
- 361 drafts (expanded from 260 with 6 new keywords), 403 authors, 1,262 ideas, 12 gaps
- Blog series (8 posts planned), reports, arXiv paper figures
- Agent team infrastructure (CLAUDE.md, scripts, dev journal)
- 5 new DB tables, schema migration, ~15 new query methods

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-04 00:48:57 +01:00
parent be9cf9c5d9
commit d6beb9c0a0
87 changed files with 24471 additions and 401 deletions

View File

@@ -0,0 +1,239 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Explorer - Living Standards Observatory</title>
<link rel="stylesheet" href="../assets/style.css">
</head>
<body>
<div class="header">
<div class="container">
<h1>Living Standards Observatory</h1>
<nav>
<a href="../index.html">Dashboard</a>
<a href="explorer.html" class="active">Explorer</a>
<a href="gaps.html">Gaps</a>
<a href="timeline.html">Timeline</a>
</nav>
</div>
</div>
<div class="container">
<div class="controls">
<div class="controls-row">
<input type="text" class="search-box" id="searchBox" placeholder="Search by name, title, summary, or keyword...">
<select id="sourceFilter" style="padding:8px;border:1px solid var(--border);border-radius:6px;font-size:0.85rem">
<option value="">All sources</option>
</select>
<div class="slider-group">Min score: <input type="range" id="minScore" min="1" max="5" step="0.1" value="1"><span class="slider-val" id="minScoreVal">1.0</span></div>
<div class="slider-group">Min novelty: <input type="range" id="minNovelty" min="1" max="5" step="1" value="1"><span class="slider-val" id="minNoveltyVal">1</span></div>
<div class="slider-group">Max overlap: <input type="range" id="maxOverlap" min="1" max="5" step="1" value="5"><span class="slider-val" id="maxOverlapVal">5</span></div>
<button class="reset-btn" onclick="resetFilters()">Reset</button>
</div>
<div class="controls-row">
<div class="chip-row" id="catChips"></div>
</div>
</div>
<div class="result-count" id="resultCount"></div>
<table>
<thead>
<tr>
<th onclick="sortBy('score')" width="60">Score <span class="sort-arrow" id="sort-score"></span></th>
<th onclick="sortBy('name')">Draft <span class="sort-arrow" id="sort-name"></span></th>
<th onclick="sortBy('source')" width="60">Src <span class="sort-arrow" id="sort-source"></span></th>
<th onclick="sortBy('date')" width="90">Date <span class="sort-arrow" id="sort-date"></span></th>
<th onclick="sortBy('novelty')" width="30">N</th>
<th onclick="sortBy('maturity')" width="30">M</th>
<th onclick="sortBy('overlap')" width="30">O</th>
<th onclick="sortBy('momentum')" width="30">Mom</th>
<th onclick="sortBy('relevance')" width="30">R</th>
<th>Categories</th>
</tr>
</thead>
<tbody id="tableBody"></tbody>
</table>
</div>
<script>
let DRAFTS = [];
let ALL_CATS = [];
let activeCats = new Set();
let sortField = 'score';
let sortAsc = false;
let expandedRow = null;
function escHtml(s) { const d = document.createElement('div'); d.textContent = s || ''; return d.innerHTML; }
function scoreBadge(s) {
const cls = s >= 4.0 ? 'score-high' : s >= 3.0 ? 'score-mid' : 'score-low';
return '<span class="score-badge ' + cls + '">' + s.toFixed(1) + '</span>';
}
function dimBar(v) { return '<span class="bar" style="width:' + (v * 12) + 'px"></span> ' + v; }
const searchBox = document.getElementById('searchBox');
const sourceFilter = document.getElementById('sourceFilter');
const minScore = document.getElementById('minScore');
const minNovelty = document.getElementById('minNovelty');
const maxOverlap = document.getElementById('maxOverlap');
searchBox.oninput = render;
sourceFilter.onchange = render;
minScore.oninput = () => { document.getElementById('minScoreVal').textContent = parseFloat(minScore.value).toFixed(1); render(); };
minNovelty.oninput = () => { document.getElementById('minNoveltyVal').textContent = minNovelty.value; render(); };
maxOverlap.oninput = () => { document.getElementById('maxOverlapVal').textContent = maxOverlap.value; render(); };
function resetFilters() {
searchBox.value = '';
sourceFilter.value = '';
minScore.value = 1; document.getElementById('minScoreVal').textContent = '1.0';
minNovelty.value = 1; document.getElementById('minNoveltyVal').textContent = '1';
maxOverlap.value = 5; document.getElementById('maxOverlapVal').textContent = '5';
activeCats.clear();
document.querySelectorAll('.chip').forEach(c => c.classList.remove('active'));
sortField = 'score'; sortAsc = false;
render();
}
function sortBy(field) {
if (sortField === field) sortAsc = !sortAsc;
else { sortField = field; sortAsc = field === 'name' || field === 'date'; }
render();
}
function cmp(a, b) {
let va = a[sortField], vb = b[sortField];
if (typeof va === 'string') return sortAsc ? va.localeCompare(vb) : vb.localeCompare(va);
return sortAsc ? va - vb : vb - va;
}
function render() {
const q = searchBox.value.toLowerCase().trim();
const src = sourceFilter.value;
const ms = parseFloat(minScore.value);
const mn = parseInt(minNovelty.value);
const mo = parseInt(maxOverlap.value);
let filtered = DRAFTS.filter(d => {
if (d.score < ms) return false;
if (d.novelty < mn) return false;
if (d.overlap > mo) return false;
if (src && (d.source || 'ietf') !== src) return false;
if (activeCats.size > 0 && !d.categories.some(c => activeCats.has(c))) return false;
if (q) {
const hay = (d.name + ' ' + d.title + ' ' + d.summary + ' ' + d.categories.join(' ')).toLowerCase();
const words = q.split(/\s+/);
if (!words.every(w => hay.includes(w))) return false;
}
return true;
});
filtered.sort(cmp);
document.querySelectorAll('.sort-arrow').forEach(el => el.textContent = '');
const arrow = document.getElementById('sort-' + sortField);
if (arrow) arrow.textContent = sortAsc ? '\u25B2' : '\u25BC';
const tbody = document.getElementById('tableBody');
tbody.innerHTML = '';
expandedRow = null;
filtered.forEach(d => {
const tr = document.createElement('tr');
tr.className = 'clickable';
const srcClass = 'source-' + (d.source || 'ietf');
tr.innerHTML =
'<td>' + scoreBadge(d.score) + '</td>' +
'<td style="max-width:300px"><a href="' + escHtml(d.url) + '" target="_blank" onclick="event.stopPropagation()" style="color:var(--accent);font-weight:500">' + escHtml(d.name) + '</a>' +
'<br><span class="dim">' + escHtml(d.title.substring(0, 80)) + '</span></td>' +
'<td><span class="source-badge ' + srcClass + '">' + (d.source || 'ietf').toUpperCase() + '</span></td>' +
'<td class="dim">' + d.date + '</td>' +
'<td>' + dimBar(d.novelty) + '</td>' +
'<td>' + dimBar(d.maturity) + '</td>' +
'<td>' + dimBar(d.overlap) + '</td>' +
'<td>' + dimBar(d.momentum) + '</td>' +
'<td>' + dimBar(d.relevance) + '</td>' +
'<td>' + d.categories.map(c => '<span class="cat-badge">' + escHtml(c) + '</span>').join('') + '</td>';
tr.onclick = () => toggleDetail(tr, d);
tbody.appendChild(tr);
});
document.getElementById('resultCount').textContent =
'Showing ' + filtered.length + ' of ' + DRAFTS.length + ' drafts';
}
function toggleDetail(tr, d) {
if (expandedRow) {
expandedRow.previousElementSibling?.classList.remove('expanded');
expandedRow.remove();
if (expandedRow._draftName === d.name) { expandedRow = null; return; }
}
tr.classList.add('expanded');
const detail = document.createElement('tr');
detail.className = 'detail-row';
detail._draftName = d.name;
function detailItem(label, score, note) {
return '<div class="detail-item"><strong>' + label + ':</strong> ' + score + '/5 ' +
'<span class="bar" style="width:' + (score * 16) + 'px"></span>' +
(note ? '<div class="note">' + escHtml(note) + '</div>' : '') + '</div>';
}
detail.innerHTML = '<td colspan="10">' +
'<div class="summary-text"><strong>Summary:</strong> ' + escHtml(d.summary) + '</div>' +
'<div class="detail-grid" style="margin-top:10px">' +
detailItem('Novelty', d.novelty, d.novelty_note) +
detailItem('Maturity', d.maturity, d.maturity_note) +
detailItem('Overlap', d.overlap, d.overlap_note) +
detailItem('Momentum', d.momentum, d.momentum_note) +
detailItem('Relevance', d.relevance, d.relevance_note) +
'<div class="detail-item"><strong>Source:</strong> ' + (d.source || 'ietf').toUpperCase() + ' &middot; <strong>Pages:</strong> ' + d.pages + '</div>' +
'</div>' +
'<div style="margin-top:8px"><a href="' + escHtml(d.url) + '" target="_blank" style="color:var(--accent)">Open document \u2192</a></div>' +
'</td>';
tr.after(detail);
expandedRow = detail;
}
async function init() {
DRAFTS = await fetch('../data/drafts.json').then(r => r.json());
// Build categories
const catSet = new Set();
const sources = new Set();
DRAFTS.forEach(d => {
d.categories.forEach(c => catSet.add(c));
sources.add(d.source || 'ietf');
});
ALL_CATS = [...catSet].sort();
// Source filter options
sources.forEach(s => {
const opt = document.createElement('option');
opt.value = s;
opt.textContent = s.toUpperCase();
sourceFilter.appendChild(opt);
});
// Category chips
const chipBox = document.getElementById('catChips');
ALL_CATS.forEach(cat => {
const el = document.createElement('span');
el.className = 'chip';
const count = DRAFTS.filter(d => d.categories.includes(cat)).length;
el.innerHTML = escHtml(cat) + '<span style="font-size:0.65rem;opacity:0.7;margin-left:2px">(' + count + ')</span>';
el.onclick = () => {
if (activeCats.has(cat)) { activeCats.delete(cat); el.classList.remove('active'); }
else { activeCats.add(cat); el.classList.add('active'); }
render();
};
chipBox.appendChild(el);
});
render();
}
init();
</script>
</body>
</html>

118
docs/observatory/gaps.html Normal file
View File

@@ -0,0 +1,118 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Gaps - Living Standards Observatory</title>
<link rel="stylesheet" href="../assets/style.css">
</head>
<body>
<div class="header">
<div class="container">
<h1>Living Standards Observatory</h1>
<nav>
<a href="../index.html">Dashboard</a>
<a href="explorer.html">Explorer</a>
<a href="gaps.html" class="active">Gaps</a>
<a href="timeline.html">Timeline</a>
</nav>
</div>
</div>
<div class="container">
<h2 style="margin-bottom:16px">Coverage Gaps</h2>
<p class="dim" style="margin-bottom:20px">Areas, problems, or technical challenges not adequately addressed by existing standards documents.</p>
<div class="controls">
<div class="controls-row">
<select id="sevFilter" style="padding:8px;border:1px solid var(--border);border-radius:6px;font-size:0.85rem">
<option value="">All severities</option>
<option value="critical">Critical</option>
<option value="high">High</option>
<option value="medium">Medium</option>
<option value="low">Low</option>
</select>
<input type="text" class="search-box" id="gapSearch" placeholder="Filter gaps..." style="max-width:400px">
</div>
</div>
<div id="gapsList"></div>
<h2 style="margin:32px 0 16px">Gap History</h2>
<p class="dim" style="margin-bottom:20px">How gaps have evolved across observatory snapshots.</p>
<div class="panel">
<table>
<thead>
<tr><th>Snapshot</th><th>Topic</th><th>Severity</th><th>Status</th></tr>
</thead>
<tbody id="historyBody"></tbody>
</table>
</div>
</div>
<script>
function escHtml(s) { const d = document.createElement('div'); d.textContent = s || ''; return d.innerHTML; }
let GAPS_DATA = null;
function renderGaps() {
const sev = document.getElementById('sevFilter').value;
const q = document.getElementById('gapSearch').value.toLowerCase().trim();
const list = document.getElementById('gapsList');
list.innerHTML = '';
let current = GAPS_DATA.current || [];
if (sev) current = current.filter(g => g.severity === sev);
if (q) current = current.filter(g => (g.topic + ' ' + g.description + ' ' + (g.category || '')).toLowerCase().includes(q));
if (current.length === 0) {
list.innerHTML = '<p class="dim" style="padding:16px">No gaps match the current filters.</p>';
return;
}
const order = {'critical': 0, 'high': 1, 'medium': 2, 'low': 3};
current.sort((a, b) => (order[a.severity] || 2) - (order[b.severity] || 2));
current.forEach(g => {
const cls = (g.severity === 'critical' || g.severity === 'high') ? g.severity : '';
list.innerHTML +=
'<div class="gap-card ' + cls + '">' +
'<h3>' + escHtml(g.topic) + '</h3>' +
'<p>' + escHtml(g.description) + '</p>' +
'<div class="meta">' +
'<span class="sev-' + g.severity + '">' + (g.severity || 'medium').toUpperCase() + '</span>' +
(g.category ? ' &middot; ' + escHtml(g.category) : '') +
(g.evidence ? '<br><em>' + escHtml(g.evidence) + '</em>' : '') +
'</div></div>';
});
}
async function init() {
GAPS_DATA = await fetch('../data/gaps.json').then(r => r.json());
document.getElementById('sevFilter').onchange = renderGaps;
document.getElementById('gapSearch').oninput = renderGaps;
renderGaps();
// History table
const history = GAPS_DATA.history || [];
const tbody = document.getElementById('historyBody');
if (history.length === 0) {
tbody.innerHTML = '<tr><td colspan="4" class="dim">No history recorded yet.</td></tr>';
} else {
history.slice(-50).reverse().forEach(h => {
const tr = document.createElement('tr');
tr.innerHTML =
'<td class="dim">' + (h.snapshot_at || h.recorded_at || '').substring(0, 10) + '</td>' +
'<td>' + escHtml(h.gap_topic) + '</td>' +
'<td><span class="sev-' + (h.severity || 'medium') + '">' + (h.severity || 'medium').toUpperCase() + '</span></td>' +
'<td>' + escHtml(h.status || 'open') + '</td>';
tbody.appendChild(tr);
});
}
}
init();
</script>
</body>
</html>

View File

@@ -0,0 +1,157 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Timeline - Living Standards Observatory</title>
<link rel="stylesheet" href="../assets/style.css">
<style>
.tl-row { display: flex; align-items: center; gap: 8px; padding: 6px 0; border-bottom: 1px solid #f0f0f0; }
.tl-month { min-width: 80px; font-size: 0.82rem; color: var(--text-dim); font-family: monospace; }
.tl-bars { flex: 1; display: flex; gap: 1px; align-items: center; }
.tl-count { min-width: 30px; text-align: right; font-size: 0.78rem; color: var(--text-dim); }
.legend { display: flex; gap: 16px; flex-wrap: wrap; margin-bottom: 16px; }
.legend-item { display: flex; align-items: center; gap: 4px; font-size: 0.8rem; }
.legend-swatch { width: 14px; height: 14px; border-radius: 3px; }
.view-toggle { display: flex; gap: 8px; margin-bottom: 16px; }
.view-btn { padding: 6px 16px; border: 1px solid var(--border); border-radius: 6px; background: var(--card-bg); cursor: pointer; font-size: 0.82rem; }
.view-btn.active { background: var(--accent); color: #fff; border-color: var(--accent); }
</style>
</head>
<body>
<div class="header">
<div class="container">
<h1>Living Standards Observatory</h1>
<nav>
<a href="../index.html">Dashboard</a>
<a href="explorer.html">Explorer</a>
<a href="gaps.html">Gaps</a>
<a href="timeline.html" class="active">Timeline</a>
</nav>
</div>
</div>
<div class="container">
<h2 style="margin-bottom:8px">Submission Timeline</h2>
<p class="dim" style="margin-bottom:20px">Monthly document submissions across standards bodies and categories.</p>
<div class="view-toggle">
<button class="view-btn active" id="btnSource" onclick="setView('source')">By Source</button>
<button class="view-btn" id="btnCategory" onclick="setView('category')">By Category</button>
</div>
<div class="legend" id="legend"></div>
<div class="chart-container" id="timeline"></div>
<div class="panel">
<div class="panel-header">Monthly Totals</div>
<table>
<thead><tr><th>Month</th><th>Total</th><th id="breakdownHeader">By Source</th></tr></thead>
<tbody id="monthTable"></tbody>
</table>
</div>
</div>
<script>
function escHtml(s) { const d = document.createElement('div'); d.textContent = s || ''; return d.innerHTML; }
const COLORS_SOURCE = {'ietf': '#4a6cf7', 'w3c': '#ef4444', 'ieee': '#10b981', 'other': '#9ca3af'};
const COLORS_CAT = [
'#636EFA', '#EF553B', '#00CC96', '#AB63FA', '#FFA15A',
'#19D3F3', '#FF6692', '#B6E880', '#FF97FF', '#FECB52',
'#7C8CF5', '#FF8C69', '#66CDAA', '#BA55D3', '#FFD700',
];
let TL_DATA = null;
let currentView = 'source';
function setView(view) {
currentView = view;
document.getElementById('btnSource').className = 'view-btn' + (view === 'source' ? ' active' : '');
document.getElementById('btnCategory').className = 'view-btn' + (view === 'category' ? ' active' : '');
document.getElementById('breakdownHeader').textContent = view === 'source' ? 'By Source' : 'By Category';
renderTimeline();
}
function renderTimeline() {
if (!TL_DATA) return;
const months = TL_DATA.months;
const isSource = currentView === 'source';
const dataMap = isSource ? TL_DATA.by_source : TL_DATA.by_category;
const keys = isSource ? TL_DATA.sources : TL_DATA.categories;
// Assign colors
const colorMap = {};
if (isSource) {
keys.forEach(k => { colorMap[k] = COLORS_SOURCE[k] || '#9ca3af'; });
} else {
keys.forEach((k, i) => { colorMap[k] = COLORS_CAT[i % COLORS_CAT.length]; });
}
// Max for scaling
let maxTotal = 0;
months.forEach(m => {
const d = dataMap[m] || {};
let t = 0;
keys.forEach(k => { t += d[k] || 0; });
if (t > maxTotal) maxTotal = t;
});
const scale = maxTotal > 0 ? 500 / maxTotal : 1;
// Legend
const legendEl = document.getElementById('legend');
legendEl.innerHTML = '';
keys.forEach(k => {
legendEl.innerHTML += '<div class="legend-item"><div class="legend-swatch" style="background:' + colorMap[k] + '"></div>' + escHtml(k) + '</div>';
});
// Chart
const container = document.getElementById('timeline');
container.innerHTML = '';
months.forEach(m => {
const d = dataMap[m] || {};
let total = 0;
keys.forEach(k => { total += d[k] || 0; });
let barsHtml = '';
keys.forEach(k => {
const v = d[k] || 0;
if (v > 0) {
const w = Math.max(v * scale, 2);
barsHtml += '<div class="tl-bar" style="width:' + w + 'px;background:' + colorMap[k] + '" title="' + escHtml(k) + ': ' + v + '"></div>';
}
});
container.innerHTML += '<div class="tl-row"><span class="tl-month">' + m + '</span><div class="tl-bars">' + barsHtml + '</div><span class="tl-count">' + total + '</span></div>';
});
// Table
const tbody = document.getElementById('monthTable');
tbody.innerHTML = '';
[...months].reverse().forEach(m => {
const d = dataMap[m] || {};
let total = 0;
const parts = [];
keys.forEach(k => {
const v = d[k] || 0;
total += v;
if (v > 0) parts.push(k + ': ' + v);
});
if (total > 0) {
const tr = document.createElement('tr');
tr.innerHTML = '<td class="dim">' + m + '</td><td>' + total + '</td><td class="dim">' + parts.join(', ') + '</td>';
tbody.appendChild(tr);
}
});
}
async function init() {
TL_DATA = await fetch('../data/timeline.json').then(r => r.json());
renderTimeline();
}
init();
</script>
</body>
</html>