feat(survey): add IETF landscape survey (kappa, phase0, rerate), gaps update; bump wimse-ect; gitignore run logs
This commit is contained in:
226
scripts/rerate-intercoder.py
Normal file
226
scripts/rerate-intercoder.py
Normal file
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inter-coder re-rating for the IETF AI/agent landscape survey.
|
||||
|
||||
Re-rates the clean IETF corpus (source='ietf', not false-positive) with TWO
|
||||
models (Sonnet + Haiku) using the EXACT pinned production prompt
|
||||
(``RATE_PROMPT_COMPACT``, abstract[:2000]) via the Anthropic Batch API (50% off).
|
||||
|
||||
Safety / reproducibility:
|
||||
- Does NOT touch the production ``ratings`` table. Output goes to
|
||||
``data/rerate/<model-alias>.jsonl`` (one JSON object per draft).
|
||||
- Batch IDs are persisted to ``data/rerate/batches.json`` so the run resumes.
|
||||
- Idempotent: drafts already present in the output JSONL are skipped on re-submit.
|
||||
|
||||
Usage:
|
||||
PYTHONPATH=src python3 scripts/rerate-intercoder.py --dry-run # cost estimate, submit nothing
|
||||
PYTHONPATH=src python3 scripts/rerate-intercoder.py --submit # create batches
|
||||
PYTHONPATH=src python3 scripts/rerate-intercoder.py --collect # poll + write results
|
||||
PYTHONPATH=src python3 scripts/rerate-intercoder.py --run # submit then collect (blocking)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from anthropic import Anthropic
|
||||
|
||||
from ietf_analyzer.analyzer import (
|
||||
CATEGORIES_SHORT,
|
||||
RATE_PROMPT_COMPACT,
|
||||
_doc_type_label,
|
||||
)
|
||||
from ietf_analyzer.config import Config
|
||||
from ietf_analyzer.db import Database
|
||||
|
||||
OUT_DIR = Path("data/rerate")
|
||||
BATCH_FILE = OUT_DIR / "batches.json"
|
||||
MAX_TOKENS = 512
|
||||
|
||||
# Anthropic batch pricing is 50% of standard. Standard (USD per 1M tokens):
|
||||
PRICING = { # input, output
|
||||
"sonnet": (3.00, 15.00),
|
||||
"haiku": (1.00, 5.00),
|
||||
}
|
||||
MODELS = {
|
||||
"sonnet": lambda c: c.claude_model,
|
||||
"haiku": lambda c: c.claude_model_cheap,
|
||||
}
|
||||
|
||||
|
||||
def clean_ietf_drafts(db: Database):
|
||||
"""The survey corpus: source='ietf', not flagged false-positive."""
|
||||
rows = db.conn.execute(
|
||||
"""SELECT d.name FROM drafts d JOIN ratings r ON d.name = r.draft_name
|
||||
WHERE d.source = 'ietf'
|
||||
AND (r.false_positive = 0 OR r.false_positive IS NULL)
|
||||
ORDER BY d.name"""
|
||||
).fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
def build_prompt(db: Database, name: str) -> str | None:
|
||||
d = db.get_draft(name)
|
||||
if d is None:
|
||||
return None
|
||||
return RATE_PROMPT_COMPACT.format(
|
||||
doc_type=_doc_type_label(d.source),
|
||||
name=d.name, title=d.title, time=d.date,
|
||||
pages=d.pages or "?",
|
||||
abstract=d.abstract[:2000],
|
||||
categories=", ".join(CATEGORIES_SHORT),
|
||||
)
|
||||
|
||||
|
||||
def already_done(alias: str) -> set[str]:
|
||||
p = OUT_DIR / f"{alias}.jsonl"
|
||||
if not p.exists():
|
||||
return set()
|
||||
done = set()
|
||||
for line in p.read_text().splitlines():
|
||||
if line.strip():
|
||||
done.add(json.loads(line)["draft_name"])
|
||||
return done
|
||||
|
||||
|
||||
def load_batches() -> dict:
|
||||
if BATCH_FILE.exists():
|
||||
return json.loads(BATCH_FILE.read_text())
|
||||
return {}
|
||||
|
||||
|
||||
def save_batches(b: dict):
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
BATCH_FILE.write_text(json.dumps(b, indent=2))
|
||||
|
||||
|
||||
def cmd_dry_run(db: Database, cfg: Config, names: list[str]):
|
||||
total_in = 0
|
||||
sample = None
|
||||
for n in names:
|
||||
p = build_prompt(db, n)
|
||||
if p is None:
|
||||
continue
|
||||
total_in += len(p) // 4 # ~4 chars/token
|
||||
if sample is None:
|
||||
sample = p
|
||||
out_est = len(names) * 350 # observed compact-JSON output size
|
||||
print(f"corpus: {len(names)} clean IETF drafts")
|
||||
print(f"est input tokens/draft: ~{total_in // max(len(names),1)}")
|
||||
print(f"est total input tokens: ~{total_in:,} | output: ~{out_est:,}")
|
||||
print("\nestimated cost (Batch API = 50% of standard):")
|
||||
grand = 0.0
|
||||
for alias, (pin, pout) in PRICING.items():
|
||||
c = (total_in / 1e6 * pin + out_est / 1e6 * pout) * 0.5
|
||||
grand += c
|
||||
print(f" {alias:7} ({MODELS[alias](cfg)}): ${c:.2f}")
|
||||
print(f" {'BOTH':7}: ${grand:.2f}")
|
||||
print(f"\n--- sample prompt ({len(sample)} chars) ---\n{sample[:600]}\n...")
|
||||
|
||||
|
||||
def cmd_submit(db: Database, cfg: Config, names: list[str], client: Anthropic):
|
||||
batches = load_batches()
|
||||
for alias in MODELS:
|
||||
if batches.get(alias, {}).get("id"):
|
||||
print(f"[{alias}] batch already submitted: {batches[alias]['id']} — skip")
|
||||
continue
|
||||
done = already_done(alias)
|
||||
todo = [n for n in names if n not in done]
|
||||
if not todo:
|
||||
print(f"[{alias}] all {len(names)} already collected — nothing to submit")
|
||||
continue
|
||||
model = MODELS[alias](cfg)
|
||||
requests = []
|
||||
for n in todo:
|
||||
p = build_prompt(db, n)
|
||||
if p is None:
|
||||
continue
|
||||
requests.append({
|
||||
"custom_id": n,
|
||||
"params": {
|
||||
"model": model,
|
||||
"max_tokens": MAX_TOKENS,
|
||||
"messages": [{"role": "user", "content": p}],
|
||||
},
|
||||
})
|
||||
batch = client.messages.batches.create(requests=requests)
|
||||
batches[alias] = {"id": batch.id, "model": model, "count": len(requests)}
|
||||
save_batches(batches)
|
||||
print(f"[{alias}] submitted {len(requests)} requests → batch {batch.id}")
|
||||
|
||||
|
||||
def cmd_collect(client: Anthropic, poll: bool):
|
||||
batches = load_batches()
|
||||
if not batches:
|
||||
print("no batches submitted yet (run --submit)")
|
||||
return False
|
||||
all_done = True
|
||||
for alias, info in batches.items():
|
||||
bid = info["id"]
|
||||
b = client.messages.batches.retrieve(bid)
|
||||
print(f"[{alias}] {bid}: {b.processing_status} "
|
||||
f"(succeeded={b.request_counts.succeeded}, errored={b.request_counts.errored}, "
|
||||
f"processing={b.request_counts.processing})")
|
||||
if b.processing_status != "ended":
|
||||
all_done = False
|
||||
continue
|
||||
out_path = OUT_DIR / f"{alias}.jsonl"
|
||||
done = already_done(alias)
|
||||
n_new = 0
|
||||
with out_path.open("a") as f:
|
||||
for result in client.messages.batches.results(bid):
|
||||
cid = result.custom_id
|
||||
if cid in done:
|
||||
continue
|
||||
if result.result.type != "succeeded":
|
||||
f.write(json.dumps({"draft_name": cid, "error": result.result.type}) + "\n")
|
||||
continue
|
||||
msg = result.result.message
|
||||
raw = msg.content[0].text
|
||||
rec = {
|
||||
"draft_name": cid,
|
||||
"model": info["model"],
|
||||
"raw": raw,
|
||||
"in_tok": msg.usage.input_tokens,
|
||||
"out_tok": msg.usage.output_tokens,
|
||||
}
|
||||
f.write(json.dumps(rec) + "\n")
|
||||
n_new += 1
|
||||
print(f"[{alias}] wrote {n_new} new results → {out_path}")
|
||||
return all_done
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--submit", action="store_true")
|
||||
ap.add_argument("--collect", action="store_true")
|
||||
ap.add_argument("--run", action="store_true", help="submit then poll-collect until done")
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = Config.load()
|
||||
db = Database(cfg)
|
||||
names = clean_ietf_drafts(db)
|
||||
|
||||
if args.dry_run:
|
||||
cmd_dry_run(db, cfg, names)
|
||||
return
|
||||
|
||||
client = Anthropic()
|
||||
if args.submit or args.run:
|
||||
cmd_submit(db, cfg, names, client)
|
||||
if args.collect:
|
||||
cmd_collect(client, poll=False)
|
||||
if args.run:
|
||||
while True:
|
||||
if cmd_collect(client, poll=True):
|
||||
print("all batches ended.")
|
||||
break
|
||||
print("...waiting 60s for batches to finish")
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
206
scripts/survey-kappa.py
Normal file
206
scripts/survey-kappa.py
Normal file
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inter-rater reliability for the IETF landscape survey.
|
||||
|
||||
Reads the two re-rating result files written by rerate-intercoder.py
|
||||
(data/rerate/sonnet.jsonl, data/rerate/haiku.jsonl) plus the existing
|
||||
production labels (ratings table), and reports:
|
||||
|
||||
- Cohen's kappa on PRIMARY CATEGORY (nominal) for each rater pair
|
||||
- Quadratic-weighted kappa on each ordinal dimension (novelty, maturity,
|
||||
overlap, momentum, relevance)
|
||||
- Raw agreement %, and the category confusion (most-confused pairs)
|
||||
|
||||
Pairs compared: sonnet-rerate vs haiku-rerate (the controlled inter-coder run),
|
||||
and each rerate vs the existing production labels (drift check).
|
||||
|
||||
Pure stdlib + numpy. No API calls. Read-only on the DB.
|
||||
Writes data/reports/survey-kappa.md.
|
||||
|
||||
Usage: PYTHONPATH=src python3 scripts/survey-kappa.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ietf_analyzer.config import Config
|
||||
from ietf_analyzer.db import Database
|
||||
|
||||
RERATE_DIR = Path("data/rerate")
|
||||
OUT = Path("data/reports/survey-kappa.md")
|
||||
DIMS = ["novelty", "maturity", "overlap", "momentum", "relevance"]
|
||||
DIM_KEYS = {"novelty": ("n", "novelty"), "maturity": ("m", "maturity"),
|
||||
"overlap": ("o", "overlap"), "momentum": ("mo", "momentum"),
|
||||
"relevance": ("r", "relevance")}
|
||||
|
||||
|
||||
def _strip_fence(text: str) -> str:
|
||||
t = text.strip()
|
||||
if t.startswith("```"):
|
||||
t = t.split("\n", 1)[1] if "\n" in t else t[3:]
|
||||
if t.rstrip().endswith("```"):
|
||||
t = t.rstrip()[:-3]
|
||||
return t.strip()
|
||||
|
||||
|
||||
def _clamp(v):
|
||||
try:
|
||||
v = int(round(float(v)))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return min(5, max(1, v))
|
||||
|
||||
|
||||
def parse_rerate(alias: str) -> dict[str, dict]:
|
||||
"""draft_name -> {primary_cat, dims...} from a rerate jsonl."""
|
||||
out = {}
|
||||
p = RERATE_DIR / f"{alias}.jsonl"
|
||||
if not p.exists():
|
||||
return out
|
||||
for line in p.read_text().splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
rec = json.loads(line)
|
||||
if "raw" not in rec:
|
||||
continue
|
||||
try:
|
||||
data = json.loads(_strip_fence(rec["raw"]))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
cats = data.get("c", data.get("categories", []))
|
||||
primary = cats[0] if isinstance(cats, list) and cats else None
|
||||
entry = {"primary": primary}
|
||||
for dim, (k1, k2) in DIM_KEYS.items():
|
||||
entry[dim] = _clamp(data.get(k1, data.get(k2)))
|
||||
out[rec["draft_name"]] = entry
|
||||
return out
|
||||
|
||||
|
||||
def load_prod(db: Database) -> dict[str, dict]:
|
||||
out = {}
|
||||
rows = db.conn.execute(
|
||||
"""SELECT r.draft_name, r.categories, r.novelty, r.maturity,
|
||||
r.overlap, r.momentum, r.relevance
|
||||
FROM drafts d JOIN ratings r ON d.name = r.draft_name
|
||||
WHERE d.source='ietf' AND (r.false_positive=0 OR r.false_positive IS NULL)"""
|
||||
).fetchall()
|
||||
for name, cats_json, *dimvals in rows:
|
||||
try:
|
||||
cats = json.loads(cats_json) if cats_json else []
|
||||
except json.JSONDecodeError:
|
||||
cats = []
|
||||
e = {"primary": cats[0] if cats else None}
|
||||
for dim, v in zip(DIMS, dimvals):
|
||||
e[dim] = _clamp(v)
|
||||
out[name] = e
|
||||
return out
|
||||
|
||||
|
||||
def cohen_kappa(a: list, b: list) -> tuple[float, float, int]:
|
||||
"""Nominal Cohen's kappa. Returns (kappa, raw_agreement, n)."""
|
||||
labels = sorted(set(a) | set(b))
|
||||
idx = {l: i for i, l in enumerate(labels)}
|
||||
k = len(labels)
|
||||
m = np.zeros((k, k))
|
||||
for x, y in zip(a, b):
|
||||
m[idx[x], idx[y]] += 1
|
||||
n = m.sum()
|
||||
po = np.trace(m) / n
|
||||
pe = (m.sum(0) @ m.sum(1)) / (n * n)
|
||||
kappa = (po - pe) / (1 - pe) if (1 - pe) else 1.0
|
||||
return kappa, po, int(n)
|
||||
|
||||
|
||||
def weighted_kappa(a: list, b: list, k: int = 5) -> tuple[float, float, int]:
|
||||
"""Quadratic-weighted kappa for ordinal 1..k ratings."""
|
||||
pairs = [(x, y) for x, y in zip(a, b) if x is not None and y is not None]
|
||||
if not pairs:
|
||||
return float("nan"), float("nan"), 0
|
||||
a2, b2 = zip(*pairs)
|
||||
o = np.zeros((k, k))
|
||||
for x, y in pairs:
|
||||
o[x - 1, y - 1] += 1
|
||||
n = o.sum()
|
||||
w = np.zeros((k, k))
|
||||
for i in range(k):
|
||||
for j in range(k):
|
||||
w[i, j] = (i - j) ** 2 / (k - 1) ** 2
|
||||
ha = np.array([a2.count(v) for v in range(1, k + 1)], float)
|
||||
hb = np.array([b2.count(v) for v in range(1, k + 1)], float)
|
||||
e = np.outer(ha, hb) / n
|
||||
num = (w * o).sum()
|
||||
den = (w * e).sum()
|
||||
kappa = 1 - num / den if den else 1.0
|
||||
raw_agree = np.trace(o) / n
|
||||
return kappa, raw_agree, int(n)
|
||||
|
||||
|
||||
def interpret(k: float) -> str:
|
||||
if k != k:
|
||||
return "n/a"
|
||||
if k < 0: return "worse than chance"
|
||||
if k < 0.20: return "slight"
|
||||
if k < 0.40: return "fair"
|
||||
if k < 0.60: return "moderate"
|
||||
if k < 0.80: return "substantial"
|
||||
return "almost perfect"
|
||||
|
||||
|
||||
def compare(name: str, A: dict, B: dict, lines: list):
|
||||
shared = sorted(set(A) & set(B))
|
||||
lines.append(f"\n## {name} (n shared = {len(shared)})\n")
|
||||
# primary category
|
||||
pa = [A[d]["primary"] for d in shared if A[d]["primary"] and B[d]["primary"]]
|
||||
pb = [B[d]["primary"] for d in shared if A[d]["primary"] and B[d]["primary"]]
|
||||
kappa, po, n = cohen_kappa(pa, pb)
|
||||
lines.append(f"**Primary category** (Cohen's κ): κ = {kappa:.3f} ({interpret(kappa)}), "
|
||||
f"raw agreement {po:.1%}, n = {n}\n")
|
||||
# confusion: top disagreements
|
||||
dis = Counter()
|
||||
for d in shared:
|
||||
x, y = A[d]["primary"], B[d]["primary"]
|
||||
if x and y and x != y:
|
||||
dis[tuple(sorted((x, y)))] += 1
|
||||
if dis:
|
||||
lines.append("\nMost-confused category pairs:\n")
|
||||
lines.append("| A | B | count |\n|---|---|------:|\n")
|
||||
for (x, y), c in dis.most_common(8):
|
||||
lines.append(f"| {x} | {y} | {c} |\n")
|
||||
# ordinal dims
|
||||
lines.append("\n**Ordinal dimensions** (quadratic-weighted κ):\n\n")
|
||||
lines.append("| dimension | κ_w | raw agree | n |\n|---|---:|---:|---:|\n")
|
||||
for dim in DIMS:
|
||||
a = [A[d][dim] for d in shared]
|
||||
b = [B[d][dim] for d in shared]
|
||||
kw, ra, n = weighted_kappa(a, b)
|
||||
lines.append(f"| {dim} | {kw:.3f} ({interpret(kw)}) | {ra:.1%} | {n} |\n")
|
||||
|
||||
|
||||
def main():
|
||||
db = Database(Config.load())
|
||||
sonnet = parse_rerate("sonnet")
|
||||
haiku = parse_rerate("haiku")
|
||||
prod = load_prod(db)
|
||||
print(f"parsed: sonnet={len(sonnet)} haiku={len(haiku)} prod={len(prod)}")
|
||||
if not sonnet or not haiku:
|
||||
print("rerate files incomplete — run rerate-intercoder.py --collect first")
|
||||
return
|
||||
|
||||
lines = ["# Inter-rater reliability — IETF landscape survey\n",
|
||||
f"\nCorpus: clean IETF (n≈524). Sonnet={len(sonnet)}, Haiku={len(haiku)}, prod labels={len(prod)}.\n",
|
||||
"\nκ interpretation (Landis & Koch): <0.2 slight, 0.2–0.4 fair, "
|
||||
"0.4–0.6 moderate, 0.6–0.8 substantial, >0.8 almost perfect.\n"]
|
||||
compare("Sonnet (re-rate) vs Haiku (re-rate) — controlled inter-coder", sonnet, haiku, lines)
|
||||
compare("Sonnet (re-rate) vs Production labels — drift/stability", sonnet, prod, lines)
|
||||
compare("Haiku (re-rate) vs Production labels", haiku, prod, lines)
|
||||
|
||||
OUT.write_text("".join(lines))
|
||||
print(f"wrote {OUT}")
|
||||
print("".join(lines))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
457
scripts/survey-phase0.py
Normal file
457
scripts/survey-phase0.py
Normal file
@@ -0,0 +1,457 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
survey-phase0.py — Deterministic, FREE (no LLM API) Phase-0 analytics for the
|
||||
quantitative survey paper of the IETF AI/agent Internet-Draft landscape.
|
||||
|
||||
CORPUS: "clean IETF corpus" = drafts where source='ietf'
|
||||
AND (ratings.false_positive=0 OR NULL). Join drafts d with ratings r
|
||||
on d.name=r.draft_name. This is 524 drafts.
|
||||
|
||||
Computes:
|
||||
1. Author / working-group concentration
|
||||
2. Embedding overlap / redundancy (pairwise cosine, nomic-embed-text 768-d)
|
||||
3. Category coverage map (primary category from ratings.categories JSON)
|
||||
|
||||
Read-only on data/drafts.db. Writes data/reports/survey-phase0.md and prints a
|
||||
headline summary to stdout. Pure stdlib + numpy only.
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
import statistics
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import numpy as np
|
||||
|
||||
DB = "data/drafts.db"
|
||||
OUT = "data/reports/survey-phase0.md"
|
||||
CLEAN = "d.source='ietf' AND (r.false_positive=0 OR r.false_positive IS NULL)"
|
||||
|
||||
# Well-known IETF Datatracker group IDs. ID 1027 is the "none" pseudo-group used
|
||||
# for individually-submitted drafts (not adopted by a WG). Others are real WGs/RGs.
|
||||
GROUP_ID_NAMES = {
|
||||
"1027": "(individual / no WG)",
|
||||
}
|
||||
|
||||
|
||||
def conn():
|
||||
c = sqlite3.connect(DB)
|
||||
c.row_factory = sqlite3.Row
|
||||
return c
|
||||
|
||||
|
||||
def gid(group_uri):
|
||||
"""Extract numeric group id from a group_uri like /api/v1/group/group/1027/ ."""
|
||||
if not group_uri:
|
||||
return None
|
||||
parts = [p for p in group_uri.strip("/").split("/") if p]
|
||||
return parts[-1] if parts else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. AUTHOR / WORKING-GROUP CONCENTRATION
|
||||
# ---------------------------------------------------------------------------
|
||||
def section_authors(c):
|
||||
total_clean = c.execute(
|
||||
f"SELECT COUNT(*) FROM drafts d JOIN ratings r ON d.name=r.draft_name WHERE {CLEAN}"
|
||||
).fetchone()[0]
|
||||
|
||||
# Top authors by # of clean-IETF drafts. Join via draft_authors.person_id -> authors.person_id.
|
||||
author_counts = c.execute(
|
||||
f"""
|
||||
SELECT a.person_id, a.name,
|
||||
COUNT(DISTINCT d.name) AS n
|
||||
FROM drafts d
|
||||
JOIN ratings r ON d.name = r.draft_name
|
||||
JOIN draft_authors da ON da.draft_name = d.name
|
||||
JOIN authors a ON a.person_id = da.person_id
|
||||
WHERE {CLEAN}
|
||||
GROUP BY a.person_id
|
||||
ORDER BY n DESC, a.name ASC
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
total_authors = len(author_counts)
|
||||
top15 = author_counts[:15]
|
||||
top10 = author_counts[:10]
|
||||
|
||||
# Concentration: share of (author, draft) memberships covered by top-10 authors,
|
||||
# and share of distinct drafts touched by at least one top-10 author.
|
||||
total_memberships = sum(a["n"] for a in author_counts)
|
||||
top10_memberships = sum(a["n"] for a in top10)
|
||||
|
||||
top10_ids = [a["person_id"] for a in top10]
|
||||
if top10_ids:
|
||||
placeholders = ",".join("?" * len(top10_ids))
|
||||
top10_drafts = c.execute(
|
||||
f"""
|
||||
SELECT COUNT(DISTINCT d.name)
|
||||
FROM drafts d
|
||||
JOIN ratings r ON d.name = r.draft_name
|
||||
JOIN draft_authors da ON da.draft_name = d.name
|
||||
WHERE {CLEAN} AND da.person_id IN ({placeholders})
|
||||
""",
|
||||
top10_ids,
|
||||
).fetchone()[0]
|
||||
else:
|
||||
top10_drafts = 0
|
||||
|
||||
drafts_with_authors = c.execute(
|
||||
f"""
|
||||
SELECT COUNT(DISTINCT d.name)
|
||||
FROM drafts d
|
||||
JOIN ratings r ON d.name = r.draft_name
|
||||
JOIN draft_authors da ON da.draft_name = d.name
|
||||
WHERE {CLEAN}
|
||||
"""
|
||||
).fetchone()[0]
|
||||
|
||||
# Working groups. drafts.group text column is unpopulated for this corpus, so
|
||||
# report it (per spec) AND the richer group_uri-based breakdown.
|
||||
raw_group = c.execute(
|
||||
f"""
|
||||
SELECT d."group" AS g, COUNT(*) AS n
|
||||
FROM drafts d JOIN ratings r ON d.name=r.draft_name
|
||||
WHERE {CLEAN}
|
||||
GROUP BY d."group" ORDER BY n DESC
|
||||
"""
|
||||
).fetchall()
|
||||
no_group_raw = c.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) FROM drafts d JOIN ratings r ON d.name=r.draft_name
|
||||
WHERE {CLEAN} AND (d."group" IS NULL OR d."group"='' OR d."group"='none')
|
||||
"""
|
||||
).fetchone()[0]
|
||||
|
||||
uri_rows = c.execute(
|
||||
f"""
|
||||
SELECT d.group_uri AS uri, COUNT(*) AS n
|
||||
FROM drafts d JOIN ratings r ON d.name=r.draft_name
|
||||
WHERE {CLEAN}
|
||||
GROUP BY d.group_uri ORDER BY n DESC
|
||||
"""
|
||||
).fetchall()
|
||||
wg_breakdown = []
|
||||
no_wg_uri = 0 # drafts in the "individual / no WG" pseudo-group or null uri
|
||||
for row in uri_rows:
|
||||
g = gid(row["uri"])
|
||||
label = GROUP_ID_NAMES.get(g, f"group/{g}" if g else "(no group_uri)")
|
||||
if g == "1027" or g is None:
|
||||
no_wg_uri += row["n"]
|
||||
wg_breakdown.append((label, g, row["n"]))
|
||||
|
||||
return {
|
||||
"total_clean": total_clean,
|
||||
"total_authors": total_authors,
|
||||
"top15": top15,
|
||||
"total_memberships": total_memberships,
|
||||
"top10_memberships": top10_memberships,
|
||||
"top10_drafts": top10_drafts,
|
||||
"drafts_with_authors": drafts_with_authors,
|
||||
"raw_group": raw_group,
|
||||
"no_group_raw": no_group_raw,
|
||||
"wg_breakdown": wg_breakdown,
|
||||
"no_wg_uri": no_wg_uri,
|
||||
"distinct_uris": len(uri_rows),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. EMBEDDING OVERLAP / REDUNDANCY
|
||||
# ---------------------------------------------------------------------------
|
||||
def load_embeddings(c):
|
||||
"""Return (names, matrix L2-normalized, titles dict) for clean corpus."""
|
||||
rows = c.execute(
|
||||
f"""
|
||||
SELECT d.name AS name, d.title AS title, em.vector AS vec
|
||||
FROM drafts d
|
||||
JOIN ratings r ON d.name = r.draft_name
|
||||
JOIN embeddings em ON em.draft_name = d.name
|
||||
WHERE {CLEAN}
|
||||
ORDER BY d.name
|
||||
"""
|
||||
).fetchall()
|
||||
names, titles, vectors = [], {}, []
|
||||
for row in rows:
|
||||
v = row["vec"]
|
||||
# Vector serialization: float32 BLOB (verified). Fall back to JSON if needed.
|
||||
try:
|
||||
arr = np.frombuffer(v, dtype=np.float32)
|
||||
if arr.size == 0:
|
||||
raise ValueError
|
||||
except Exception:
|
||||
arr = np.asarray(json.loads(v), dtype=np.float32)
|
||||
names.append(row["name"])
|
||||
titles[row["name"]] = row["title"]
|
||||
vectors.append(arr)
|
||||
mat = np.vstack(vectors).astype(np.float64)
|
||||
norms = np.linalg.norm(mat, axis=1, keepdims=True)
|
||||
norms[norms == 0] = 1.0
|
||||
mat = mat / norms
|
||||
return names, mat, titles
|
||||
|
||||
|
||||
def section_embeddings(c):
|
||||
names, mat, titles = load_embeddings(c)
|
||||
n = len(names)
|
||||
sim = mat @ mat.T # cosine since rows are unit-normalized
|
||||
iu, ju = np.triu_indices(n, k=1)
|
||||
off = sim[iu, ju]
|
||||
|
||||
dist = {
|
||||
"n_drafts": n,
|
||||
"n_pairs": int(off.size),
|
||||
"dim": mat.shape[1],
|
||||
"mean": float(np.mean(off)),
|
||||
"median": float(np.median(off)),
|
||||
"p90": float(np.percentile(off, 90)),
|
||||
"p99": float(np.percentile(off, 99)),
|
||||
"max": float(np.max(off)),
|
||||
}
|
||||
|
||||
# Top 20 most-similar pairs
|
||||
order = np.argsort(off)[::-1][:20]
|
||||
top_pairs = []
|
||||
for k in order:
|
||||
a, b = names[iu[k]], names[ju[k]]
|
||||
top_pairs.append((a, b, float(off[k]), titles.get(a, ""), titles.get(b, "")))
|
||||
|
||||
# Drafts with at least one near-duplicate (cosine > 0.9)
|
||||
thr = 0.9
|
||||
near = sim > thr
|
||||
np.fill_diagonal(near, False)
|
||||
has_near = int(np.sum(near.any(axis=1)))
|
||||
n_near_pairs = int(np.sum(off > thr))
|
||||
|
||||
dist["near_dup_drafts"] = has_near
|
||||
dist["near_dup_pairs"] = n_near_pairs
|
||||
return dist, top_pairs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. CATEGORY COVERAGE MAP
|
||||
# ---------------------------------------------------------------------------
|
||||
def section_categories(c):
|
||||
rows = c.execute(
|
||||
f"""
|
||||
SELECT d.name AS name, r.categories AS cats,
|
||||
r.novelty, r.maturity, r.overlap, r.momentum, r.relevance
|
||||
FROM drafts d JOIN ratings r ON d.name = r.draft_name
|
||||
WHERE {CLEAN}
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
primary_counts = Counter()
|
||||
multi_cat = 0
|
||||
cat_dims = defaultdict(lambda: {"novelty": [], "maturity": [], "overlap": [],
|
||||
"momentum": [], "relevance": [], "composite": []})
|
||||
n_total = 0
|
||||
n_no_cat = 0
|
||||
|
||||
for row in rows:
|
||||
n_total += 1
|
||||
cats = []
|
||||
if row["cats"]:
|
||||
try:
|
||||
cats = [x for x in json.loads(row["cats"]) if x]
|
||||
except Exception:
|
||||
cats = []
|
||||
if not cats:
|
||||
primary = "(uncategorized)"
|
||||
n_no_cat += 1
|
||||
else:
|
||||
primary = cats[0]
|
||||
if len(cats) > 1:
|
||||
multi_cat += 1
|
||||
primary_counts[primary] += 1
|
||||
|
||||
dims = cat_dims[primary]
|
||||
vals = []
|
||||
for d in ("novelty", "maturity", "overlap", "momentum", "relevance"):
|
||||
if row[d] is not None:
|
||||
dims[d].append(row[d])
|
||||
vals.append(row[d])
|
||||
if vals:
|
||||
dims["composite"].append(statistics.mean(vals))
|
||||
|
||||
# Build per-category summary
|
||||
cat_summary = []
|
||||
for cat, cnt in primary_counts.most_common():
|
||||
dims = cat_dims[cat]
|
||||
def m(key):
|
||||
return round(statistics.mean(dims[key]), 2) if dims[key] else None
|
||||
cat_summary.append({
|
||||
"cat": cat, "n": cnt,
|
||||
"relevance": m("relevance"),
|
||||
"composite": m("composite"),
|
||||
"novelty": m("novelty"), "maturity": m("maturity"),
|
||||
"overlap": m("overlap"), "momentum": m("momentum"),
|
||||
})
|
||||
|
||||
sparsest = sorted([s for s in cat_summary if s["cat"] != "(uncategorized)"],
|
||||
key=lambda s: s["n"])[:3]
|
||||
|
||||
return {
|
||||
"n_total": n_total,
|
||||
"primary_counts": primary_counts,
|
||||
"cat_summary": cat_summary,
|
||||
"multi_cat": multi_cat,
|
||||
"n_no_cat": n_no_cat,
|
||||
"sparsest": sparsest,
|
||||
"sum_check": sum(primary_counts.values()),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# REPORT
|
||||
# ---------------------------------------------------------------------------
|
||||
def md_table(headers, rows):
|
||||
out = ["| " + " | ".join(headers) + " |",
|
||||
"| " + " | ".join("---" for _ in headers) + " |"]
|
||||
for r in rows:
|
||||
out.append("| " + " | ".join(str(x) for x in r) + " |")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def build_report(au, em, cat):
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
L = []
|
||||
A = L.append
|
||||
A("# Survey Phase 0 — Quantitative IETF AI/Agent Draft Landscape\n")
|
||||
A(f"_Generated {ts} by `scripts/survey-phase0.py` (deterministic, no LLM API calls)._\n")
|
||||
A("**Corpus definition (\"clean IETF corpus\"):** `source='ietf' AND (ratings.false_positive=0 OR NULL)`, "
|
||||
f"joining `drafts d` with `ratings r` on `d.name=r.draft_name`. **N = {au['total_clean']} drafts.**\n")
|
||||
|
||||
# 1
|
||||
A("## 1. Author / Working-Group Concentration\n")
|
||||
A(f"- Distinct authors across the clean corpus: **{au['total_authors']}**")
|
||||
A(f"- Clean drafts with at least one resolved author in `draft_authors`: **{au['drafts_with_authors']}** "
|
||||
f"of {au['total_clean']} "
|
||||
f"({100*au['drafts_with_authors']/au['total_clean']:.1f}%); the remainder have no rows in `draft_authors`.")
|
||||
A(f"- Total (author, draft) authorship memberships: **{au['total_memberships']}**\n")
|
||||
|
||||
A("### Top 15 authors by clean-IETF draft count\n")
|
||||
A(md_table(["#", "Author", "Drafts"],
|
||||
[(i + 1, a["name"], a["n"]) for i, a in enumerate(au["top15"])]))
|
||||
A("")
|
||||
share_mem = 100 * au["top10_memberships"] / au["total_memberships"] if au["total_memberships"] else 0
|
||||
share_drafts = 100 * au["top10_drafts"] / au["total_clean"]
|
||||
A("### Concentration (top 10 authors)\n")
|
||||
A(f"- Top-10 authors account for **{au['top10_memberships']} of {au['total_memberships']} authorship "
|
||||
f"memberships ({share_mem:.1f}%)**.")
|
||||
A(f"- Top-10 authors appear on **{au['top10_drafts']} of {au['total_clean']} distinct clean drafts "
|
||||
f"({share_drafts:.1f}%)** (a draft is counted once even if multiple top-10 authors appear on it).\n")
|
||||
|
||||
A("### Working groups\n")
|
||||
A("The `drafts.group` text column is effectively unpopulated for this corpus "
|
||||
f"(only {au['no_group_raw']} drafts resolve to no/empty/'none' group, and the column has no real WG names). "
|
||||
"The meaningful grouping signal is `drafts.group_uri` (Datatracker group IDs).\n")
|
||||
A(f"- Drafts with no working group (group/1027 \"individual\" pseudo-group or null `group_uri`): "
|
||||
f"**{au['no_wg_uri']}** of {au['total_clean']}.")
|
||||
A(f"- Distinct `group_uri` values: **{au['distinct_uris']}**.\n")
|
||||
A("Top 15 groups by `group_uri` (clean-IETF draft count):\n")
|
||||
A(md_table(["#", "Group (Datatracker ID / label)", "Drafts"],
|
||||
[(i + 1, lbl, n) for i, (lbl, g, n) in enumerate(au["wg_breakdown"][:15])]))
|
||||
A("")
|
||||
A("> Note: ID 1027 is the Datatracker \"none\" pseudo-group for individually submitted drafts; "
|
||||
"it is not a real working group. Real WG/RG IDs are the remaining rows.\n")
|
||||
|
||||
# 2
|
||||
A("## 2. Embedding Overlap / Redundancy\n")
|
||||
A(f"- Embedding model: `nomic-embed-text`, dimension **{em['dim']}**, stored as float32 BLOB.")
|
||||
A(f"- Drafts embedded (clean corpus): **{em['n_drafts']}** (100% coverage); "
|
||||
f"pairwise comparisons: **{em['n_pairs']:,}**.\n")
|
||||
A("### Pairwise cosine similarity (off-diagonal)\n")
|
||||
A(md_table(["Mean", "Median", "p90", "p99", "Max"],
|
||||
[(f"{em['mean']:.4f}", f"{em['median']:.4f}", f"{em['p90']:.4f}",
|
||||
f"{em['p99']:.4f}", f"{em['max']:.4f}")]))
|
||||
A("")
|
||||
A(f"- Near-duplicate pairs (cosine > 0.9): **{em['near_dup_pairs']}**.")
|
||||
A(f"- Drafts with at least one near-duplicate (cosine > 0.9): **{em['near_dup_drafts']}** "
|
||||
f"of {em['n_drafts']} ({100*em['near_dup_drafts']/em['n_drafts']:.1f}%).\n")
|
||||
A("### Top 20 most-similar draft pairs\n")
|
||||
rows = []
|
||||
for i, (a, b, cos, ta, tb) in enumerate(em["pairs"]):
|
||||
rows.append((i + 1, f"{cos:.4f}", f"`{a}`<br>`{b}`",
|
||||
f"{(ta or '')[:70]}<br>{(tb or '')[:70]}"))
|
||||
A(md_table(["#", "Cosine", "Draft A / Draft B", "Title A / Title B"], rows))
|
||||
A("")
|
||||
|
||||
# 3
|
||||
A("## 3. Category Coverage Map\n")
|
||||
A(f"Primary category = first element of the `ratings.categories` JSON array. "
|
||||
f"Sum check: **{cat['sum_check']} = {au['total_clean']}** "
|
||||
f"({'OK' if cat['sum_check'] == au['total_clean'] else 'MISMATCH'}).\n")
|
||||
A(f"- Drafts carrying more than one category: **{cat['multi_cat']}** of {cat['n_total']} "
|
||||
f"({100*cat['multi_cat']/cat['n_total']:.1f}%).")
|
||||
A(f"- Drafts with no category (uncategorized): **{cat['n_no_cat']}**.\n")
|
||||
A("### Primary-category distribution + mean scores\n")
|
||||
A("Scores are 1-5 integers from `ratings`. `composite` = mean of the 5 dimensions "
|
||||
"(novelty, maturity, overlap, momentum, relevance) per draft, averaged over the category.\n")
|
||||
rows = []
|
||||
for s in cat["cat_summary"]:
|
||||
rows.append((s["cat"], s["n"], f"{100*s['n']/cat['n_total']:.1f}%",
|
||||
s["relevance"], s["composite"], s["novelty"], s["maturity"],
|
||||
s["overlap"], s["momentum"]))
|
||||
A(md_table(["Category", "N", "Share", "Relevance", "Composite",
|
||||
"Novelty", "Maturity", "Overlap", "Momentum"], rows))
|
||||
A("")
|
||||
A("### Three sparsest categories (descriptive)\n")
|
||||
A("Listed neutrally as the categories with the fewest drafts in this corpus:\n")
|
||||
A(md_table(["Category", "N", "Relevance", "Composite"],
|
||||
[(s["cat"], s["n"], s["relevance"], s["composite"]) for s in cat["sparsest"]]))
|
||||
A("")
|
||||
|
||||
return "\n".join(L) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
c = conn()
|
||||
try:
|
||||
au = section_authors(c)
|
||||
dist, pairs = section_embeddings(c)
|
||||
dist["pairs"] = pairs
|
||||
cat = section_categories(c)
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
report = build_report(au, dist, cat)
|
||||
import os
|
||||
os.makedirs("data/reports", exist_ok=True)
|
||||
with open(OUT, "w") as f:
|
||||
f.write(report)
|
||||
|
||||
# Internal consistency checks
|
||||
assert cat["sum_check"] == au["total_clean"], "category sum != 524"
|
||||
share_drafts = 100 * au["top10_drafts"] / au["total_clean"]
|
||||
assert share_drafts <= 100, "author draft share > 100%"
|
||||
|
||||
# Headline summary
|
||||
print("=" * 60)
|
||||
print("SURVEY PHASE 0 — HEADLINE NUMBERS")
|
||||
print("=" * 60)
|
||||
print(f"Clean IETF corpus: {au['total_clean']} drafts")
|
||||
print(f"Distinct authors: {au['total_authors']}")
|
||||
print(f"Drafts with authors: {au['drafts_with_authors']} ({100*au['drafts_with_authors']/au['total_clean']:.1f}%)")
|
||||
print(f"Top-10 author draft share: {share_drafts:.1f}%")
|
||||
print(f"Drafts with no WG: {au['no_wg_uri']} (group_uri-based)")
|
||||
print(f"Distinct group_uris: {au['distinct_uris']}")
|
||||
print("-" * 60)
|
||||
print(f"Embeddings: {dist['n_drafts']} drafts, dim {dist['dim']}, {dist['n_pairs']:,} pairs")
|
||||
print(f"Cosine mean/median: {dist['mean']:.4f} / {dist['median']:.4f}")
|
||||
print(f"Cosine p90/p99/max: {dist['p90']:.4f} / {dist['p99']:.4f} / {dist['max']:.4f}")
|
||||
print(f"Near-dup pairs (>0.9): {dist['near_dup_pairs']}")
|
||||
print(f"Drafts w/ near-dup (>0.9): {dist['near_dup_drafts']}")
|
||||
print("-" * 60)
|
||||
print(f"Primary categories: {len(cat['primary_counts'])} (sum={cat['sum_check']})")
|
||||
print(f"Multi-category drafts: {cat['multi_cat']} ({100*cat['multi_cat']/cat['n_total']:.1f}%)")
|
||||
print(f"Uncategorized drafts: {cat['n_no_cat']}")
|
||||
top3 = cat["cat_summary"][:3]
|
||||
print("Top 3 categories: " + ", ".join(f"{s['cat']} ({s['n']})" for s in top3))
|
||||
print("Sparsest 3 categories: " + ", ".join(f"{s['cat']} ({s['n']})" for s in cat["sparsest"]))
|
||||
print("=" * 60)
|
||||
print(f"Report written to {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user