feat: add cross-run finding regression detection
This commit is contained in:
@@ -11,6 +11,7 @@
|
||||
# ./lib/archeflow-memory.sh list # List all active lessons
|
||||
# ./lib/archeflow-memory.sh decay # Apply decay to all lessons
|
||||
# ./lib/archeflow-memory.sh forget <id> # Archive a lesson by ID
|
||||
# ./lib/archeflow-memory.sh regression-check <events> # Detect regressions from previously fixed findings
|
||||
#
|
||||
# Dependencies: jq, bash 4+
|
||||
|
||||
@@ -361,6 +362,86 @@ cmd_audit_check() {
|
||||
done <<< "$lesson_ids"
|
||||
}
|
||||
|
||||
cmd_regression_check() {
|
||||
local events_file="${1:?Usage: $0 regression-check <events.jsonl>}"
|
||||
|
||||
if [[ ! -f "$events_file" ]]; then
|
||||
echo "Error: events file not found: $events_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract current run_id
|
||||
local run_id
|
||||
run_id=$(jq -r '.run_id' "$events_file" | head -1)
|
||||
|
||||
# Find the previous run from index.jsonl
|
||||
local INDEX_FILE=".archeflow/events/index.jsonl"
|
||||
if [[ ! -f "$INDEX_FILE" ]]; then
|
||||
echo "[archeflow-memory] No index.jsonl found — skipping regression check." >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
local prev_run_id
|
||||
prev_run_id=$(jq -r --arg rid "$run_id" 'select(.run_id != $rid) | .run_id' "$INDEX_FILE" 2>/dev/null | tail -1)
|
||||
|
||||
if [[ -z "$prev_run_id" ]]; then
|
||||
echo "[archeflow-memory] No previous run found — skipping regression check." >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
local prev_events=".archeflow/events/${prev_run_id}.jsonl"
|
||||
if [[ ! -f "$prev_events" ]]; then
|
||||
echo "[archeflow-memory] Previous run events not found: $prev_events" >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Extract resolved findings from previous run (fix.applied events)
|
||||
local resolved_findings
|
||||
resolved_findings=$(jq -r 'select(.type == "fix.applied") | .data.finding // empty' "$prev_events" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$resolved_findings" ]]; then
|
||||
echo "[archeflow-memory] No resolved findings in previous run — nothing to regress." >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Extract current run findings from review.verdict events
|
||||
local current_findings
|
||||
current_findings=$(jq -r '
|
||||
select(.type == "review.verdict") |
|
||||
.data.findings[]? | .description // empty
|
||||
' "$events_file" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$current_findings" ]]; then
|
||||
echo "[archeflow-memory] No findings in current run — no regressions." >&2
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Compare: for each resolved finding, check if it reappeared
|
||||
local regressions=0
|
||||
while IFS= read -r resolved; do
|
||||
[[ -z "$resolved" ]] && continue
|
||||
|
||||
while IFS= read -r current; do
|
||||
[[ -z "$current" ]] && continue
|
||||
local overlap
|
||||
overlap=$(keyword_overlap "$resolved" "$current")
|
||||
if [[ "$overlap" -ge 50 ]]; then
|
||||
echo "REGRESSION: \"$resolved\" (fixed in $prev_run_id) reappeared as \"$current\""
|
||||
regressions=$((regressions + 1))
|
||||
break
|
||||
fi
|
||||
done <<< "$current_findings"
|
||||
done <<< "$resolved_findings"
|
||||
|
||||
if [[ "$regressions" -gt 0 ]]; then
|
||||
echo "[archeflow-memory] $regressions regression(s) detected from run $prev_run_id." >&2
|
||||
return 1
|
||||
else
|
||||
echo "[archeflow-memory] No regressions detected." >&2
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_add() {
|
||||
local type="${1:-preference}"
|
||||
local desc="${2:-}"
|
||||
@@ -503,6 +584,7 @@ if [[ $# -lt 1 ]]; then
|
||||
echo " decay Apply decay to all lessons" >&2
|
||||
echo " forget <id> Archive a lesson by ID" >&2
|
||||
echo " audit-check <run_id> Check lesson effectiveness for a run" >&2
|
||||
echo " regression-check <events.jsonl> Detect regressions from previously fixed findings" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -535,6 +617,10 @@ case "$COMMAND" in
|
||||
[[ $# -lt 1 ]] && { echo "Usage: $0 audit-check <run_id>" >&2; exit 1; }
|
||||
cmd_audit_check "$1"
|
||||
;;
|
||||
regression-check)
|
||||
[[ $# -lt 1 ]] && { echo "Usage: $0 regression-check <events.jsonl>" >&2; exit 1; }
|
||||
cmd_regression_check "$1"
|
||||
;;
|
||||
*)
|
||||
echo "Unknown command: $COMMAND" >&2
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user