feat: add cross-run finding regression detection

This commit is contained in:
2026-04-04 08:39:22 +02:00
parent 30ddc6a2c4
commit 5e2117c9be
2 changed files with 94 additions and 0 deletions

View File

@@ -11,6 +11,7 @@
# ./lib/archeflow-memory.sh list # List all active lessons # ./lib/archeflow-memory.sh list # List all active lessons
# ./lib/archeflow-memory.sh decay # Apply decay to all lessons # ./lib/archeflow-memory.sh decay # Apply decay to all lessons
# ./lib/archeflow-memory.sh forget <id> # Archive a lesson by ID # ./lib/archeflow-memory.sh forget <id> # Archive a lesson by ID
# ./lib/archeflow-memory.sh regression-check <events> # Detect regressions from previously fixed findings
# #
# Dependencies: jq, bash 4+ # Dependencies: jq, bash 4+
@@ -361,6 +362,86 @@ cmd_audit_check() {
done <<< "$lesson_ids" done <<< "$lesson_ids"
} }
cmd_regression_check() {
local events_file="${1:?Usage: $0 regression-check <events.jsonl>}"
if [[ ! -f "$events_file" ]]; then
echo "Error: events file not found: $events_file" >&2
exit 1
fi
# Extract current run_id
local run_id
run_id=$(jq -r '.run_id' "$events_file" | head -1)
# Find the previous run from index.jsonl
local INDEX_FILE=".archeflow/events/index.jsonl"
if [[ ! -f "$INDEX_FILE" ]]; then
echo "[archeflow-memory] No index.jsonl found — skipping regression check." >&2
return 0
fi
local prev_run_id
prev_run_id=$(jq -r --arg rid "$run_id" 'select(.run_id != $rid) | .run_id' "$INDEX_FILE" 2>/dev/null | tail -1)
if [[ -z "$prev_run_id" ]]; then
echo "[archeflow-memory] No previous run found — skipping regression check." >&2
return 0
fi
local prev_events=".archeflow/events/${prev_run_id}.jsonl"
if [[ ! -f "$prev_events" ]]; then
echo "[archeflow-memory] Previous run events not found: $prev_events" >&2
return 0
fi
# Extract resolved findings from previous run (fix.applied events)
local resolved_findings
resolved_findings=$(jq -r 'select(.type == "fix.applied") | .data.finding // empty' "$prev_events" 2>/dev/null || true)
if [[ -z "$resolved_findings" ]]; then
echo "[archeflow-memory] No resolved findings in previous run — nothing to regress." >&2
return 0
fi
# Extract current run findings from review.verdict events
local current_findings
current_findings=$(jq -r '
select(.type == "review.verdict") |
.data.findings[]? | .description // empty
' "$events_file" 2>/dev/null || true)
if [[ -z "$current_findings" ]]; then
echo "[archeflow-memory] No findings in current run — no regressions." >&2
return 0
fi
# Compare: for each resolved finding, check if it reappeared
local regressions=0
while IFS= read -r resolved; do
[[ -z "$resolved" ]] && continue
while IFS= read -r current; do
[[ -z "$current" ]] && continue
local overlap
overlap=$(keyword_overlap "$resolved" "$current")
if [[ "$overlap" -ge 50 ]]; then
echo "REGRESSION: \"$resolved\" (fixed in $prev_run_id) reappeared as \"$current\""
regressions=$((regressions + 1))
break
fi
done <<< "$current_findings"
done <<< "$resolved_findings"
if [[ "$regressions" -gt 0 ]]; then
echo "[archeflow-memory] $regressions regression(s) detected from run $prev_run_id." >&2
return 1
else
echo "[archeflow-memory] No regressions detected." >&2
return 0
fi
}
cmd_add() { cmd_add() {
local type="${1:-preference}" local type="${1:-preference}"
local desc="${2:-}" local desc="${2:-}"
@@ -503,6 +584,7 @@ if [[ $# -lt 1 ]]; then
echo " decay Apply decay to all lessons" >&2 echo " decay Apply decay to all lessons" >&2
echo " forget <id> Archive a lesson by ID" >&2 echo " forget <id> Archive a lesson by ID" >&2
echo " audit-check <run_id> Check lesson effectiveness for a run" >&2 echo " audit-check <run_id> Check lesson effectiveness for a run" >&2
echo " regression-check <events.jsonl> Detect regressions from previously fixed findings" >&2
exit 1 exit 1
fi fi
@@ -535,6 +617,10 @@ case "$COMMAND" in
[[ $# -lt 1 ]] && { echo "Usage: $0 audit-check <run_id>" >&2; exit 1; } [[ $# -lt 1 ]] && { echo "Usage: $0 audit-check <run_id>" >&2; exit 1; }
cmd_audit_check "$1" cmd_audit_check "$1"
;; ;;
regression-check)
[[ $# -lt 1 ]] && { echo "Usage: $0 regression-check <events.jsonl>" >&2; exit 1; }
cmd_regression_check "$1"
;;
*) *)
echo "Unknown command: $COMMAND" >&2 echo "Unknown command: $COMMAND" >&2
exit 1 exit 1

View File

@@ -583,6 +583,14 @@ If `CYCLE >= MAX_CYCLES` and issues remain:
./lib/archeflow-event.sh "$RUN_ID" run.complete act "" \ ./lib/archeflow-event.sh "$RUN_ID" run.complete act "" \
'{"status":"completed","cycles":<N>,"agents_total":<count>,"fixes_total":<count>,"shadows":0,"artifacts":[<list>]}' '{"status":"completed","cycles":<N>,"agents_total":<count>,"fixes_total":<count>,"shadows":0,"artifacts":[<list>]}'
# Check for regressions from previously fixed findings
if ./lib/archeflow-memory.sh regression-check ".archeflow/events/${RUN_ID}.jsonl"; then
echo "No regressions detected."
else
echo "WARNING: Regressions detected — previously fixed findings have reappeared."
echo "Review the regression output above and consider addressing them."
fi
# Generate report # Generate report
./lib/archeflow-report.sh .archeflow/events/${RUN_ID}.jsonl ./lib/archeflow-report.sh .archeflow/events/${RUN_ID}.jsonl