feat: add memory injection audit trail for effectiveness tracking

lib/archeflow-memory.sh

@@ -201,6 +201,16 @@ cmd_inject() {
   local domain="${1:-}"
   local archetype="${2:-}"
 
+  # Parse optional --audit <run_id>
+  local audit_run_id=""
+  shift 2 2>/dev/null || true
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --audit) audit_run_id="$2"; shift 2 ;;
+      *) shift ;;
+    esac
+  done
+
   if [[ ! -f "$LESSONS_FILE" ]]; then
     return 0
   fi
@@ -237,14 +247,118 @@ cmd_inject() {
     return 0
   fi
 
+  # Collect injected lesson IDs for audit
+  local injected_ids=()
+
   echo "## Known Issues (from past runs)"
   while IFS= read -r lesson; do
-    local desc freq src
+    local desc freq src lid
     desc=$(echo "$lesson" | jq -r '.description')
     freq=$(echo "$lesson" | jq -r '.frequency')
     src=$(echo "$lesson" | jq -r '.source')
+    lid=$(echo "$lesson" | jq -r '.id')
+    injected_ids+=("$lid")
     echo "- ${desc} [seen ${freq}x, ${src}]"
   done <<< "$lessons"
+
+  # Write audit record if --audit was passed
+  if [[ -n "$audit_run_id" && ${#injected_ids[@]} -gt 0 ]]; then
+    ensure_dir
+    local AUDIT_FILE="${MEMORY_DIR}/audit.jsonl"
+    local ids_json
+    ids_json=$(printf '%s\n' "${injected_ids[@]}" | jq -R . | jq -sc .)
+    jq -cn \
+      --arg ts "$(now_ts)" \
+      --arg run_id "$audit_run_id" \
+      --arg domain "$domain" \
+      --arg archetype "$archetype" \
+      --argjson lessons_injected "$ids_json" \
+      --argjson lesson_count "${#injected_ids[@]}" \
+      '{ts:$ts,run_id:$run_id,domain:$domain,archetype:$archetype,lessons_injected:$lessons_injected,lesson_count:$lesson_count}' \
+      >> "$AUDIT_FILE"
+  fi
+}
+
+cmd_audit_check() {
+  local run_id="${1:?Usage: $0 audit-check <run_id>}"
+  local AUDIT_FILE="${MEMORY_DIR}/audit.jsonl"
+  local EVENTS_FILE=".archeflow/events/${run_id}.jsonl"
+
+  if [[ ! -f "$AUDIT_FILE" ]]; then
+    echo "No audit records found." >&2
+    return 0
+  fi
+
+  if [[ ! -f "$EVENTS_FILE" ]]; then
+    echo "No events file found for run $run_id." >&2
+    return 0
+  fi
+
+  # Get lessons injected for this run
+  local injected
+  injected=$(jq -c "select(.run_id == \"$run_id\")" "$AUDIT_FILE" 2>/dev/null || true)
+
+  if [[ -z "$injected" ]]; then
+    echo "No audit records for run $run_id." >&2
+    return 0
+  fi
+
+  # Get all finding descriptions from review.verdict events
+  local finding_descs
+  finding_descs=$(jq -r '
+    select(.type == "review.verdict") |
+    .data.findings[]? | .description // empty
+  ' "$EVENTS_FILE" 2>/dev/null | tr '[:upper:]' '[:lower:]' || true)
+
+  # For each injected lesson, check if findings match the lesson's topic
+  local lesson_ids
+  lesson_ids=$(echo "$injected" | jq -r '.lessons_injected[]' 2>/dev/null | sort -u)
+
+  while IFS= read -r lid; do
+    [[ -z "$lid" ]] && continue
+
+    # Get lesson description
+    local lesson_desc
+    lesson_desc=$(jq -r "select(.id == \"$lid\") | .description" "$LESSONS_FILE" 2>/dev/null | head -1)
+    [[ -z "$lesson_desc" ]] && continue
+
+    # Check keyword overlap between lesson and findings
+    local lesson_tokens finding_overlap
+    lesson_tokens=$(tokenize "$lesson_desc")
+    finding_overlap=0
+
+    if [[ -n "$finding_descs" ]]; then
+      local finding_tokens
+      finding_tokens=$(echo "$finding_descs" | tr -cs '[:alnum:]' '\n' | awk 'length >= 3' | sort -u)
+      local common
+      common=$(comm -12 <(echo "$lesson_tokens") <(echo "$finding_tokens") | wc -l)
+      local total
+      total=$(echo "$lesson_tokens" | wc -l)
+      if [[ "$total" -gt 0 ]]; then
+        finding_overlap=$(( common * 100 / total ))
+      fi
+    fi
+
+    local effectiveness
+    if [[ "$finding_overlap" -ge 30 ]]; then
+      effectiveness="ineffective"  # Issue repeated despite lesson injection
+    else
+      effectiveness="helpful"  # Issue was prevented (no matching finding)
+    fi
+
+    # Append result to audit.jsonl
+    jq -cn \
+      --arg ts "$(now_ts)" \
+      --arg run_id "$run_id" \
+      --arg lesson_id "$lid" \
+      --arg lesson_desc "$lesson_desc" \
+      --arg effectiveness "$effectiveness" \
+      --argjson overlap "$finding_overlap" \
+      '{ts:$ts,run_id:$run_id,type:"effectiveness_check",lesson_id:$lesson_id,lesson_desc:$lesson_desc,effectiveness:$effectiveness,keyword_overlap_pct:$overlap}' \
+      >> "$AUDIT_FILE"
+
+    echo "[archeflow-memory] Lesson $lid ($effectiveness): $lesson_desc" >&2
+  done <<< "$lesson_ids"
 }
 
 cmd_add() {
@@ -383,11 +497,12 @@ if [[ $# -lt 1 ]]; then
   echo "" >&2
   echo "Commands:" >&2
   echo "  extract <events.jsonl>       Extract lessons from a completed run" >&2
-  echo "  inject <domain> <archetype>  Output relevant lessons for injection" >&2
+  echo "  inject <domain> <archetype> [--audit <run_id>]  Output relevant lessons for injection" >&2
   echo "  add <type> <description>     Manually add a lesson" >&2
   echo "  list                         List all active lessons" >&2
   echo "  decay                        Apply decay to all lessons" >&2
   echo "  forget <id>                  Archive a lesson by ID" >&2
+  echo "  audit-check <run_id>         Check lesson effectiveness for a run" >&2
   exit 1
 fi
 
@@ -416,6 +531,10 @@ case "$COMMAND" in
     [[ $# -lt 1 ]] && { echo "Usage: $0 forget <id>" >&2; exit 1; }
     cmd_forget "$1"
     ;;
+  audit-check)
+    [[ $# -lt 1 ]] && { echo "Usage: $0 audit-check <run_id>" >&2; exit 1; }
+    cmd_audit_check "$1"
+    ;;
   *)
     echo "Unknown command: $COMMAND" >&2
     exit 1