From 6a49c21bbeab71107fda67c484f38488f368daba Mon Sep 17 00:00:00 2001 From: Christian Nennemann Date: Mon, 6 Apr 2026 21:20:05 +0200 Subject: [PATCH 1/2] test: add bats test suite for lib/ helper scripts 110 tests across 10 test files covering all lib/ scripts: - archeflow-event.sh: JSONL format, seq numbering, parent fields, validation - archeflow-memory.sh: add/list/decay/forget/inject/extract commands - archeflow-git.sh: branch creation, commit format, merge strategies, safety - archeflow-report.sh: markdown output, summary mode, in-progress handling - archeflow-progress.sh: progress.md generation, JSON mode, error handling - archeflow-score.sh: archetype scoring, effectiveness report, validation - archeflow-dag.sh: DAG rendering, color flags, tree structure - archeflow-rollback.sh: arg parsing, phase validation, mutual exclusivity - archeflow-init.sh: template listing, clone from project, arg validation - archeflow-review.sh: diff modes, stats, branch/commit range review Includes test_helper.bash (shared setup/teardown with temp git repos) and scripts/run-tests.sh runner. --- scripts/run-tests.sh | 34 +++++ tests/archeflow-dag.bats | 71 +++++++++++ tests/archeflow-event.bats | 127 +++++++++++++++++++ tests/archeflow-git.bats | 212 +++++++++++++++++++++++++++++++ tests/archeflow-init.bats | 81 ++++++++++++ tests/archeflow-memory.bats | 227 ++++++++++++++++++++++++++++++++++ tests/archeflow-progress.bats | 78 ++++++++++++ tests/archeflow-report.bats | 80 ++++++++++++ tests/archeflow-review.bats | 82 ++++++++++++ tests/archeflow-rollback.bats | 58 +++++++++ tests/archeflow-score.bats | 105 ++++++++++++++++ tests/test_helper.bash | 40 ++++++ 12 files changed, 1195 insertions(+) create mode 100755 scripts/run-tests.sh create mode 100644 tests/archeflow-dag.bats create mode 100644 tests/archeflow-event.bats create mode 100644 tests/archeflow-git.bats create mode 100644 tests/archeflow-init.bats create mode 100644 tests/archeflow-memory.bats create mode 100644 tests/archeflow-progress.bats create mode 100644 tests/archeflow-report.bats create mode 100644 tests/archeflow-review.bats create mode 100644 tests/archeflow-rollback.bats create mode 100644 tests/archeflow-score.bats create mode 100644 tests/test_helper.bash diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh new file mode 100755 index 0000000..fda29d0 --- /dev/null +++ b/scripts/run-tests.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# run-tests.sh — Run all ArcheFlow bats tests. +# +# Usage: ./scripts/run-tests.sh [bats-args...] +# Examples: +# ./scripts/run-tests.sh # Run all tests +# ./scripts/run-tests.sh --filter "event" # Run only event tests +# ./scripts/run-tests.sh -t # TAP output + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +TESTS_DIR="$PROJECT_DIR/tests" + +# Find bats binary +BATS="${BATS:-}" +if [[ -z "$BATS" ]]; then + if command -v bats &>/dev/null; then + BATS="bats" + elif [[ -x "$HOME/.local/bin/bats" ]]; then + BATS="$HOME/.local/bin/bats" + else + echo "ERROR: bats not found. Install bats-core or set BATS env var." >&2 + exit 1 + fi +fi + +echo "Running ArcheFlow tests..." +echo " bats: $($BATS --version)" +echo " tests: $TESTS_DIR" +echo "" + +exec "$BATS" "$@" "$TESTS_DIR"/*.bats diff --git a/tests/archeflow-dag.bats b/tests/archeflow-dag.bats new file mode 100644 index 0000000..e2fbb3c --- /dev/null +++ b/tests/archeflow-dag.bats @@ -0,0 +1,71 @@ +# Tests for archeflow-dag.sh — ASCII DAG rendering from JSONL events. +# +# Validates: basic rendering, parent relationships, color flags, missing file handling. + +setup() { + load test_helper + _common_setup + + # Create a standard events file with parent relationships + cat > "$BATS_TEST_TMPDIR/dag-events.jsonl" <<'EVENTS' +{"ts":"2026-04-03T10:00:00Z","run_id":"dag-run","seq":1,"parent":[],"type":"run.start","phase":"plan","agent":null,"data":{"task":"DAG test"}} +{"ts":"2026-04-03T10:01:00Z","run_id":"dag-run","seq":2,"parent":[1],"type":"agent.complete","phase":"plan","agent":"creator","data":{"archetype":"creator","duration_ms":60000,"tokens":1500}} +{"ts":"2026-04-03T10:02:00Z","run_id":"dag-run","seq":3,"parent":[2],"type":"phase.transition","phase":"do","agent":null,"data":{"from":"plan","to":"do"}} +{"ts":"2026-04-03T10:03:00Z","run_id":"dag-run","seq":4,"parent":[3],"type":"agent.complete","phase":"do","agent":"maker","data":{"archetype":"maker","duration_ms":120000,"tokens":3000}} +{"ts":"2026-04-03T10:04:00Z","run_id":"dag-run","seq":5,"parent":[4],"type":"run.complete","phase":"act","agent":null,"data":{"agents_total":2,"fixes_total":0}} +EVENTS +} + +@test "dag: exits 1 with usage when called with no args" { + run "$LIB_DIR/archeflow-dag.sh" + [ "$status" -eq 1 ] + [[ "$output" == *"Usage"* ]] +} + +@test "dag: exits 1 when events file not found" { + run "$LIB_DIR/archeflow-dag.sh" nonexistent.jsonl + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] +} + +@test "dag: renders run.start as root node" { + run "$LIB_DIR/archeflow-dag.sh" "$BATS_TEST_TMPDIR/dag-events.jsonl" --no-color + [ "$status" -eq 0 ] + [[ "$output" == *"#1"* ]] + [[ "$output" == *"run.start"* ]] +} + +@test "dag: renders agent.complete events with archetype name" { + run "$LIB_DIR/archeflow-dag.sh" "$BATS_TEST_TMPDIR/dag-events.jsonl" --no-color + [ "$status" -eq 0 ] + [[ "$output" == *"creator"* ]] + [[ "$output" == *"maker"* ]] +} + +@test "dag: renders phase transitions" { + run "$LIB_DIR/archeflow-dag.sh" "$BATS_TEST_TMPDIR/dag-events.jsonl" --no-color + [ "$status" -eq 0 ] + [[ "$output" == *"plan"* ]] + [[ "$output" == *"do"* ]] +} + +@test "dag: renders run.complete with agent/fix counts" { + run "$LIB_DIR/archeflow-dag.sh" "$BATS_TEST_TMPDIR/dag-events.jsonl" --no-color + [ "$status" -eq 0 ] + [[ "$output" == *"run.complete"* ]] + [[ "$output" == *"2 agents"* ]] +} + +@test "dag: --no-color suppresses ANSI codes" { + run "$LIB_DIR/archeflow-dag.sh" "$BATS_TEST_TMPDIR/dag-events.jsonl" --no-color + [ "$status" -eq 0 ] + # Should not contain escape sequences + [[ "$output" != *$'\033'* ]] +} + +@test "dag: uses tree-drawing characters for hierarchy" { + run "$LIB_DIR/archeflow-dag.sh" "$BATS_TEST_TMPDIR/dag-events.jsonl" --no-color + [ "$status" -eq 0 ] + # Should contain box-drawing characters (either unicode or ASCII connectors) + [[ "$output" == *"├"* ]] || [[ "$output" == *"└"* ]] +} diff --git a/tests/archeflow-event.bats b/tests/archeflow-event.bats new file mode 100644 index 0000000..f7a330f --- /dev/null +++ b/tests/archeflow-event.bats @@ -0,0 +1,127 @@ +# Tests for archeflow-event.sh — structured JSONL event logging. +# +# Validates: JSONL output format, sequence numbering, parent field handling, +# input validation, file/directory creation. + +setup() { + load test_helper + _common_setup +} + +teardown() { + _common_teardown +} + +@test "event: exits 1 with usage when called with fewer than 4 args" { + run "$LIB_DIR/archeflow-event.sh" run1 type1 plan + [ "$status" -eq 1 ] + [[ "$output" == *"Usage"* ]] +} + +@test "event: creates events directory and file on first call" { + run "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" '{"task":"test"}' + [ "$status" -eq 0 ] + [ -d ".archeflow/events" ] + [ -f ".archeflow/events/test-run.jsonl" ] +} + +@test "event: first event has seq=1" { + run "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" '{"task":"test"}' + [ "$status" -eq 0 ] + local seq + seq=$(head -1 ".archeflow/events/test-run.jsonl" | jq -r '.seq') + [ "$seq" -eq 1 ] +} + +@test "event: second event has seq=2" { + "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" '{"task":"test"}' 2>/dev/null + "$LIB_DIR/archeflow-event.sh" test-run agent.complete plan creator '{"dur":100}' "1" 2>/dev/null + local count + count=$(wc -l < ".archeflow/events/test-run.jsonl") + [ "$count" -eq 2 ] + local seq2 + seq2=$(tail -1 ".archeflow/events/test-run.jsonl" | jq -r '.seq') + [ "$seq2" -eq 2 ] +} + +@test "event: output is valid JSONL" { + "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" '{"task":"hello"}' 2>/dev/null + # jq will fail if the line is not valid JSON + jq empty ".archeflow/events/test-run.jsonl" +} + +@test "event: fields are correctly populated" { + "$LIB_DIR/archeflow-event.sh" test-run agent.complete do maker '{"tokens":500}' 2>/dev/null + local event + event=$(head -1 ".archeflow/events/test-run.jsonl") + [ "$(echo "$event" | jq -r '.run_id')" = "test-run" ] + [ "$(echo "$event" | jq -r '.type')" = "agent.complete" ] + [ "$(echo "$event" | jq -r '.phase')" = "do" ] + [ "$(echo "$event" | jq -r '.agent')" = "maker" ] + [ "$(echo "$event" | jq -r '.data.tokens')" = "500" ] +} + +@test "event: empty agent becomes null in JSON" { + "$LIB_DIR/archeflow-event.sh" test-run phase.transition do "" '{"from":"plan","to":"do"}' 2>/dev/null + local agent + agent=$(head -1 ".archeflow/events/test-run.jsonl" | jq -r '.agent') + [ "$agent" = "null" ] +} + +@test "event: parent field is empty array for root events" { + "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" '{}' 2>/dev/null + local parent + parent=$(head -1 ".archeflow/events/test-run.jsonl" | jq -c '.parent') + [ "$parent" = "[]" ] +} + +@test "event: single parent is parsed correctly" { + "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" '{}' 2>/dev/null + "$LIB_DIR/archeflow-event.sh" test-run agent.complete plan creator '{}' "1" 2>/dev/null + local parent + parent=$(tail -1 ".archeflow/events/test-run.jsonl" | jq -c '.parent') + [ "$parent" = "[1]" ] +} + +@test "event: multiple parents (fan-in) are parsed correctly" { + "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" '{}' 2>/dev/null + "$LIB_DIR/archeflow-event.sh" test-run a plan "" '{}' "1" 2>/dev/null + "$LIB_DIR/archeflow-event.sh" test-run b plan "" '{}' "1" 2>/dev/null + "$LIB_DIR/archeflow-event.sh" test-run merge plan "" '{}' "2,3" 2>/dev/null + local parent + parent=$(tail -1 ".archeflow/events/test-run.jsonl" | jq -c '.parent') + [ "$parent" = "[2,3]" ] +} + +@test "event: rejects invalid JSON data" { + run "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" 'not-json' + [ "$status" -eq 1 ] + [[ "$output" == *"invalid JSON"* ]] +} + +@test "event: rejects invalid parent format" { + run "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" '{}' "abc" + [ "$status" -eq 1 ] + [[ "$output" == *"invalid parent format"* ]] +} + +@test "event: timestamp is ISO 8601 UTC format" { + "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" '{}' 2>/dev/null + local ts + ts=$(head -1 ".archeflow/events/test-run.jsonl" | jq -r '.ts') + # Matches YYYY-MM-DDTHH:MM:SSZ + [[ "$ts" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] +} + +@test "event: default data is empty object when omitted" { + "$LIB_DIR/archeflow-event.sh" test-run run.start plan agent 2>/dev/null + local data + data=$(head -1 ".archeflow/events/test-run.jsonl" | jq -c '.data') + [ "$data" = "{}" ] +} + +@test "event: confirmation message goes to stderr" { + run "$LIB_DIR/archeflow-event.sh" test-run run.start plan "" '{}' "" 2>&1 + [[ "$output" == *"[archeflow-event]"* ]] + [[ "$output" == *"#1"* ]] +} diff --git a/tests/archeflow-git.bats b/tests/archeflow-git.bats new file mode 100644 index 0000000..e15a8ee --- /dev/null +++ b/tests/archeflow-git.bats @@ -0,0 +1,212 @@ +# Tests for archeflow-git.sh — git branch/commit strategy for ArcheFlow runs. +# +# Validates: branch creation with correct naming, commit formatting, +# merge strategies, input validation, and safety guards. + +setup() { + load test_helper + _common_setup +} + +teardown() { + _common_teardown +} + +# --- Usage --- + +@test "git: exits 1 with usage when called with fewer than 2 args" { + run "$LIB_DIR/archeflow-git.sh" + [ "$status" -eq 1 ] + [[ "$output" == *"Usage"* ]] +} + +@test "git: exits 1 for unknown command" { + run "$LIB_DIR/archeflow-git.sh" nonexistent test-run + [ "$status" -ne 0 ] + [[ "$output" == *"Unknown command"* ]] +} + +# --- init --- + +@test "git init: creates branch with archeflow/ prefix" { + run "$LIB_DIR/archeflow-git.sh" init test-run + [ "$status" -eq 0 ] + local current + current=$(git branch --show-current) + [ "$current" = "archeflow/test-run" ] +} + +@test "git init: stores base branch in .archeflow/runs//base-branch" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + [ -f ".archeflow/runs/test-run/base-branch" ] + local base + base=$(cat ".archeflow/runs/test-run/base-branch") + [ "$base" = "main" ] +} + +@test "git init: fails if branch already exists" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + git checkout main --quiet + run "$LIB_DIR/archeflow-git.sh" init test-run + [ "$status" -ne 0 ] + [[ "$output" == *"already exists"* ]] +} + +# --- commit --- + +@test "git commit: uses conventional commit format by default" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + # Create a file to commit + mkdir -p .archeflow/events + echo '{"test":true}' > .archeflow/events/test-run.jsonl + "$LIB_DIR/archeflow-git.sh" commit test-run plan "initial plan" 2>/dev/null + local msg + msg=$(git log -1 --format=%s) + [[ "$msg" == "archeflow(plan): initial plan" ]] +} + +@test "git commit: stages event file automatically" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + mkdir -p .archeflow/events + echo '{"test":true}' > .archeflow/events/test-run.jsonl + "$LIB_DIR/archeflow-git.sh" commit test-run plan "test commit" 2>/dev/null + + # Verify the event file was committed + local committed_files + committed_files=$(git diff-tree --no-commit-id --name-only -r HEAD) + [[ "$committed_files" == *"test-run.jsonl"* ]] +} + +@test "git commit: stages extra files passed as arguments" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + echo "extra content" > extra.txt + "$LIB_DIR/archeflow-git.sh" commit test-run do "with extras" extra.txt 2>/dev/null + local committed_files + committed_files=$(git diff-tree --no-commit-id --name-only -r HEAD) + [[ "$committed_files" == *"extra.txt"* ]] +} + +@test "git commit: reports nothing to commit when no changes" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + # Commit the init artifacts first so there's a clean state + git add -A && git commit -m "init artifacts" --quiet 2>/dev/null || true + run bash -c "cd '$BATS_TEST_TMPDIR' && '$LIB_DIR/archeflow-git.sh' commit test-run plan 'empty' 2>&1" + [ "$status" -eq 0 ] + [[ "$output" == *"Nothing to commit"* ]] +} + +@test "git commit: fails if not on the run branch" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + git checkout main --quiet + run "$LIB_DIR/archeflow-git.sh" commit test-run plan "wrong branch" + [ "$status" -ne 0 ] + [[ "$output" == *"Expected to be on branch"* ]] +} + +# --- phase-commit --- + +@test "git phase-commit: creates commit with phase transition message" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + mkdir -p .archeflow/events + echo '{"test":true}' > .archeflow/events/test-run.jsonl + "$LIB_DIR/archeflow-git.sh" phase-commit test-run plan 2>/dev/null + local msg + msg=$(git log -1 --format=%s) + # Should contain the phase transition arrow + [[ "$msg" == *"plan"* ]] + [[ "$msg" == *"do"* ]] +} + +# --- merge --- + +@test "git merge: squash merge is the default strategy" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + mkdir -p .archeflow/events + echo '{"test":true}' > .archeflow/events/test-run.jsonl + "$LIB_DIR/archeflow-git.sh" commit test-run plan "test" 2>/dev/null + "$LIB_DIR/archeflow-git.sh" merge test-run 2>/dev/null + + local current + current=$(git branch --show-current) + [ "$current" = "main" ] + + local msg + msg=$(git log -1 --format=%s) + [[ "$msg" == *"archeflow run test-run"* ]] +} + +@test "git merge: --no-ff creates a merge commit" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + mkdir -p .archeflow/events + echo '{"test":true}' > .archeflow/events/test-run.jsonl + "$LIB_DIR/archeflow-git.sh" commit test-run plan "test" 2>/dev/null + "$LIB_DIR/archeflow-git.sh" merge test-run --no-ff 2>/dev/null + + local current + current=$(git branch --show-current) + [ "$current" = "main" ] + + # no-ff merge commit should have 2 parents + local parent_count + parent_count=$(git cat-file -p HEAD | grep -c '^parent') + [ "$parent_count" -eq 2 ] +} + +@test "git merge: rejects unknown merge strategy" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + mkdir -p .archeflow/events + echo '{"test":true}' > .archeflow/events/test-run.jsonl + "$LIB_DIR/archeflow-git.sh" commit test-run plan "test" 2>/dev/null + run "$LIB_DIR/archeflow-git.sh" merge test-run --fast-forward + [ "$status" -ne 0 ] + [[ "$output" == *"Unknown merge strategy"* ]] +} + +@test "git merge: fails with uncommitted changes" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + echo "dirty" > dirty.txt + git add dirty.txt + run "$LIB_DIR/archeflow-git.sh" merge test-run + [ "$status" -ne 0 ] + [[ "$output" == *"Uncommitted changes"* ]] +} + +# --- format_message --- + +@test "git commit: simple style uses 'phase: msg' format" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + # Create config with simple style + mkdir -p .archeflow + echo "commit_style: simple" > .archeflow/config.yaml + mkdir -p .archeflow/events + echo '{"test":true}' > .archeflow/events/test-run.jsonl + "$LIB_DIR/archeflow-git.sh" commit test-run plan "simple test" 2>/dev/null + local msg + msg=$(git log -1 --format=%s) + [ "$msg" = "plan: simple test" ] +} + +# --- status --- + +@test "git status: shows branch info for existing run" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + run "$LIB_DIR/archeflow-git.sh" status test-run + [ "$status" -eq 0 ] + [[ "$output" == *"Branch: archeflow/test-run"* ]] + [[ "$output" == *"Base: main"* ]] +} + +@test "git status: fails for nonexistent branch" { + run "$LIB_DIR/archeflow-git.sh" status nonexistent + [ "$status" -ne 0 ] + [[ "$output" == *"does not exist"* ]] +} + +# --- cleanup --- + +@test "git cleanup: fails if currently on the run branch" { + "$LIB_DIR/archeflow-git.sh" init test-run 2>/dev/null + run "$LIB_DIR/archeflow-git.sh" cleanup test-run + [ "$status" -ne 0 ] + [[ "$output" == *"Cannot delete"* ]] +} diff --git a/tests/archeflow-init.bats b/tests/archeflow-init.bats new file mode 100644 index 0000000..aa14db0 --- /dev/null +++ b/tests/archeflow-init.bats @@ -0,0 +1,81 @@ +# Tests for archeflow-init.sh — project initialization from templates. +# +# Validates: usage output, --list, --from (clone), and argument parsing. + +setup() { + load test_helper + _common_setup +} + +teardown() { + _common_teardown +} + +@test "init: shows usage when called with no args" { + run "$LIB_DIR/archeflow-init.sh" + [ "$status" -eq 0 ] + [[ "$output" == *"Usage"* ]] + [[ "$output" == *"bundle-name"* ]] +} + +@test "init: --list shows template listing without errors" { + run "$LIB_DIR/archeflow-init.sh" --list + [ "$status" -eq 0 ] + [[ "$output" == *"Templates"* ]] + [[ "$output" == *"Bundles"* ]] +} + +@test "init: --from fails when source has no .archeflow dir" { + local source_dir + source_dir=$(mktemp -d) + run "$LIB_DIR/archeflow-init.sh" --from "$source_dir" + [ "$status" -ne 0 ] + [[ "$output" == *"No .archeflow/"* ]] + rm -rf "$source_dir" +} + +@test "init: --from clones setup from another project" { + # Create a source project with .archeflow structure + local source_dir + source_dir=$(mktemp -d) + mkdir -p "$source_dir/.archeflow/teams" "$source_dir/.archeflow/workflows" + echo "name: test-team" > "$source_dir/.archeflow/teams/test.yaml" + echo "name: test-workflow" > "$source_dir/.archeflow/workflows/test.yaml" + echo "bundle: test" > "$source_dir/.archeflow/config.yaml" + + run "$LIB_DIR/archeflow-init.sh" --from "$source_dir" + [ "$status" -eq 0 ] + [ -f ".archeflow/teams/test.yaml" ] + [ -f ".archeflow/workflows/test.yaml" ] + [ -f ".archeflow/config.yaml" ] + rm -rf "$source_dir" +} + +@test "init: --from skips events and artifacts directories" { + local source_dir + source_dir=$(mktemp -d) + mkdir -p "$source_dir/.archeflow/events" "$source_dir/.archeflow/artifacts" + mkdir -p "$source_dir/.archeflow/teams" + echo "name: test" > "$source_dir/.archeflow/teams/t.yaml" + echo '{"test":true}' > "$source_dir/.archeflow/events/run.jsonl" + echo "artifact" > "$source_dir/.archeflow/artifacts/test.txt" + + run "$LIB_DIR/archeflow-init.sh" --from "$source_dir" + [ "$status" -eq 0 ] + [ ! -f ".archeflow/events/run.jsonl" ] + [ ! -f ".archeflow/artifacts/test.txt" ] + [[ "$output" == *"skipped events"* ]] + rm -rf "$source_dir" +} + +@test "init: rejects unknown options" { + run "$LIB_DIR/archeflow-init.sh" --nonexistent + [ "$status" -ne 0 ] + [[ "$output" == *"Unknown option"* ]] +} + +@test "init: --save fails with no .archeflow directory" { + run "$LIB_DIR/archeflow-init.sh" --save test-save + [ "$status" -ne 0 ] + [[ "$output" == *"No .archeflow/"* ]] +} diff --git a/tests/archeflow-memory.bats b/tests/archeflow-memory.bats new file mode 100644 index 0000000..74a604c --- /dev/null +++ b/tests/archeflow-memory.bats @@ -0,0 +1,227 @@ +# Tests for archeflow-memory.sh — cross-run lesson memory management. +# +# Validates: add, list, decay, forget, inject filtering, and JSONL format. + +setup() { + load test_helper + _common_setup +} + +teardown() { + _common_teardown +} + +# --- Usage / error handling --- + +@test "memory: exits 1 with usage when called with no args" { + run "$LIB_DIR/archeflow-memory.sh" + [ "$status" -eq 1 ] + [[ "$output" == *"Usage"* ]] +} + +@test "memory: exits 1 for unknown command" { + run "$LIB_DIR/archeflow-memory.sh" nonexistent + [ "$status" -eq 1 ] + [[ "$output" == *"Unknown command"* ]] +} + +# --- add --- + +@test "memory add: creates lessons.jsonl and appends a valid JSONL line" { + run "$LIB_DIR/archeflow-memory.sh" add preference "Always validate inputs" + [ "$status" -eq 0 ] + [ -f ".archeflow/memory/lessons.jsonl" ] + jq empty ".archeflow/memory/lessons.jsonl" +} + +@test "memory add: lesson has correct fields" { + "$LIB_DIR/archeflow-memory.sh" add pattern "Guardian misses SQL injection" 2>/dev/null + [ "$(jq -r '.type' .archeflow/memory/lessons.jsonl)" = "pattern" ] + [ "$(jq -r '.description' .archeflow/memory/lessons.jsonl)" = "Guardian misses SQL injection" ] + [ "$(jq -r '.source' .archeflow/memory/lessons.jsonl)" = "user_feedback" ] + [ "$(jq -r '.frequency' .archeflow/memory/lessons.jsonl)" = "1" ] + [ "$(jq -r '.run_id' .archeflow/memory/lessons.jsonl)" = "manual" ] + [ "$(jq -r '.domain' .archeflow/memory/lessons.jsonl)" = "general" ] +} + +@test "memory add: generates sequential IDs" { + "$LIB_DIR/archeflow-memory.sh" add pattern "first lesson" 2>/dev/null + "$LIB_DIR/archeflow-memory.sh" add pattern "second lesson" 2>/dev/null + local id1 id2 + id1=$(head -1 ".archeflow/memory/lessons.jsonl" | jq -r '.id') + id2=$(tail -1 ".archeflow/memory/lessons.jsonl" | jq -r '.id') + [ "$id1" = "m-001" ] + [ "$id2" = "m-002" ] +} + +@test "memory add: generates tags from description" { + "$LIB_DIR/archeflow-memory.sh" add pattern "Guardian misses SQL injection attacks" 2>/dev/null + local tags_count + tags_count=$(head -1 ".archeflow/memory/lessons.jsonl" | jq '.tags | length') + [ "$tags_count" -gt 0 ] +} + +@test "memory add: exits 1 when description is missing" { + run "$LIB_DIR/archeflow-memory.sh" add pattern + [ "$status" -eq 1 ] + [[ "$output" == *"Usage"* ]] +} + +# --- list --- + +@test "memory list: shows message when no lessons exist" { + run bash -c "'$LIB_DIR/archeflow-memory.sh' list 2>&1" + [ "$status" -eq 0 ] + [[ "$output" == *"No lessons"* ]] +} + +@test "memory list: shows table header and lesson data" { + "$LIB_DIR/archeflow-memory.sh" add pattern "Test lesson for listing" 2>/dev/null + run "$LIB_DIR/archeflow-memory.sh" list + [ "$status" -eq 0 ] + [[ "$output" == *"ID"* ]] + [[ "$output" == *"Freq"* ]] + [[ "$output" == *"m-001"* ]] + [[ "$output" == *"Test lesson for listing"* ]] +} + +# --- decay --- + +@test "memory decay: increments runs_since_last_seen" { + "$LIB_DIR/archeflow-memory.sh" add pattern "Decay test lesson" 2>/dev/null + "$LIB_DIR/archeflow-memory.sh" decay 2>/dev/null + local runs_since + runs_since=$(head -1 ".archeflow/memory/lessons.jsonl" | jq '.runs_since_last_seen') + [ "$runs_since" -eq 1 ] +} + +@test "memory decay: decrements frequency after 10 runs" { + "$LIB_DIR/archeflow-memory.sh" add pattern "Decay frequency test" 2>/dev/null + # Set frequency=3 and runs_since=9 to trigger decay on next call + local tmp=".archeflow/memory/lessons.jsonl.tmp" + head -1 ".archeflow/memory/lessons.jsonl" | jq -c '.frequency = 3 | .runs_since_last_seen = 9' > "$tmp" + mv "$tmp" ".archeflow/memory/lessons.jsonl" + + "$LIB_DIR/archeflow-memory.sh" decay 2>/dev/null + local freq + freq=$(head -1 ".archeflow/memory/lessons.jsonl" | jq '.frequency') + [ "$freq" -eq 2 ] +} + +@test "memory decay: archives lesson when frequency reaches 0" { + "$LIB_DIR/archeflow-memory.sh" add pattern "Will be archived" 2>/dev/null + # Set frequency=1 and runs_since=9 to trigger archival + local tmp=".archeflow/memory/lessons.jsonl.tmp" + head -1 ".archeflow/memory/lessons.jsonl" | jq -c '.frequency = 1 | .runs_since_last_seen = 9' > "$tmp" + mv "$tmp" ".archeflow/memory/lessons.jsonl" + + "$LIB_DIR/archeflow-memory.sh" decay 2>/dev/null + + # Lesson should be gone from lessons file (file should be empty) + local remaining + remaining=$(wc -l < ".archeflow/memory/lessons.jsonl" | tr -d ' ') + [ "$remaining" -eq 0 ] + + # And present in archive + [ -f ".archeflow/memory/archive.jsonl" ] + local archived_count + archived_count=$(wc -l < ".archeflow/memory/archive.jsonl" | tr -d ' ') + [ "$archived_count" -eq 1 ] +} + +@test "memory decay: does nothing when no lessons exist" { + run "$LIB_DIR/archeflow-memory.sh" decay + [ "$status" -eq 0 ] +} + +# --- forget --- + +@test "memory forget: moves lesson to archive" { + "$LIB_DIR/archeflow-memory.sh" add pattern "Will forget this" 2>/dev/null + "$LIB_DIR/archeflow-memory.sh" forget m-001 2>/dev/null + + # Lessons file should be empty + local remaining + remaining=$(wc -l < ".archeflow/memory/lessons.jsonl" | tr -d ' ') + [ "$remaining" -eq 0 ] + + # Archive should have it + [ -f ".archeflow/memory/archive.jsonl" ] + local archived_id + archived_id=$(head -1 ".archeflow/memory/archive.jsonl" | jq -r '.id') + [ "$archived_id" = "m-001" ] +} + +@test "memory forget: exits 1 for nonexistent ID" { + "$LIB_DIR/archeflow-memory.sh" add pattern "test" 2>/dev/null + run "$LIB_DIR/archeflow-memory.sh" forget m-999 + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] +} + +@test "memory forget: exits 1 when no lessons file exists" { + run "$LIB_DIR/archeflow-memory.sh" forget m-001 + [ "$status" -eq 1 ] + [[ "$output" == *"No lessons file"* ]] +} + +# --- inject --- + +@test "memory inject: outputs nothing when no lessons file exists" { + run "$LIB_DIR/archeflow-memory.sh" inject code guardian + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "memory inject: outputs relevant lessons with frequency >= 2" { + "$LIB_DIR/archeflow-memory.sh" add pattern "Test injection lesson" 2>/dev/null + # Bump frequency to 2 + local tmp=".archeflow/memory/lessons.jsonl.tmp" + jq -c '.frequency = 2' ".archeflow/memory/lessons.jsonl" > "$tmp" + mv "$tmp" ".archeflow/memory/lessons.jsonl" + + run "$LIB_DIR/archeflow-memory.sh" inject "" "" + [ "$status" -eq 0 ] + [[ "$output" == *"Known Issues"* ]] + [[ "$output" == *"Test injection lesson"* ]] +} + +@test "memory inject: skips lessons with frequency < 2 (except preferences)" { + "$LIB_DIR/archeflow-memory.sh" add pattern "Low frequency lesson" 2>/dev/null + # frequency is 1 by default, type is pattern -> should NOT be injected + run "$LIB_DIR/archeflow-memory.sh" inject "" "" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "memory inject: always injects preferences regardless of frequency" { + "$LIB_DIR/archeflow-memory.sh" add preference "User prefers explicit error messages" 2>/dev/null + run "$LIB_DIR/archeflow-memory.sh" inject "" "" + [ "$status" -eq 0 ] + [[ "$output" == *"User prefers explicit error messages"* ]] +} + +# --- extract --- + +@test "memory extract: exits 1 when events file not found" { + run "$LIB_DIR/archeflow-memory.sh" extract nonexistent.jsonl + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] +} + +@test "memory extract: extracts findings from review.verdict events" { + # Create a mock events file with a review.verdict + mkdir -p .archeflow/events + cat > /tmp/test-events.jsonl <<'EOF' +{"run_id":"test-run","seq":1,"type":"run.start","phase":"plan","data":{"task":"test"}} +{"run_id":"test-run","seq":2,"type":"review.verdict","phase":"check","data":{"archetype":"guardian","verdict":"needs_changes","findings":[{"severity":"warning","description":"Missing input validation on user endpoint","category":"code"}]}} +EOF + + run "$LIB_DIR/archeflow-memory.sh" extract /tmp/test-events.jsonl + [ "$status" -eq 0 ] + [ -f ".archeflow/memory/lessons.jsonl" ] + local desc + desc=$(jq -r '.description' ".archeflow/memory/lessons.jsonl") + [[ "$desc" == *"Missing input validation"* ]] + rm -f /tmp/test-events.jsonl +} diff --git a/tests/archeflow-progress.bats b/tests/archeflow-progress.bats new file mode 100644 index 0000000..52835ba --- /dev/null +++ b/tests/archeflow-progress.bats @@ -0,0 +1,78 @@ +# Tests for archeflow-progress.sh — live progress file generation. +# +# Validates: markdown output structure, JSON mode, missing events handling, exit codes. + +setup() { + load test_helper + _common_setup + + # Create standard events for progress tests + mkdir -p .archeflow/events + cat > ".archeflow/events/test-run.jsonl" <<'EVENTS' +{"ts":"2026-04-03T10:00:00Z","run_id":"test-run","seq":1,"parent":[],"type":"run.start","phase":"plan","agent":null,"data":{"task":"Build feature","workflow":"standard","team":"default"}} +{"ts":"2026-04-03T10:01:00Z","run_id":"test-run","seq":2,"parent":[1],"type":"agent.complete","phase":"plan","agent":"creator","data":{"archetype":"creator","duration_ms":60000,"tokens":1500,"estimated_cost_usd":0.02,"summary":"Planned"}} +EVENTS +} + +@test "progress: exits 1 with usage when called with no args" { + run "$LIB_DIR/archeflow-progress.sh" + [ "$status" -eq 1 ] + [[ "$output" == *"Usage"* ]] +} + +@test "progress: exits 1 when events file not found" { + run "$LIB_DIR/archeflow-progress.sh" nonexistent-run + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] +} + +@test "progress: default mode generates progress.md" { + run "$LIB_DIR/archeflow-progress.sh" test-run + [ "$status" -eq 0 ] + [ -f ".archeflow/progress.md" ] + [[ "$output" == *"# ArcheFlow Run: test-run"* ]] + [[ "$output" == *"Status:"* ]] + [[ "$output" == *"Progress"* ]] +} + +@test "progress: json mode outputs valid JSON" { + run "$LIB_DIR/archeflow-progress.sh" test-run --json + [ "$status" -eq 0 ] + echo "$output" | jq empty + local run_id + run_id=$(echo "$output" | jq -r '.run_id') + [ "$run_id" = "test-run" ] +} + +@test "progress: json mode includes completed agents" { + run "$LIB_DIR/archeflow-progress.sh" test-run --json + [ "$status" -eq 0 ] + local completed_count + completed_count=$(echo "$output" | jq '.completed | length') + [ "$completed_count" -eq 1 ] + local agent + agent=$(echo "$output" | jq -r '.completed[0].agent') + [ "$agent" = "creator" ] +} + +@test "progress: json mode shows correct phase" { + run "$LIB_DIR/archeflow-progress.sh" test-run --json + [ "$status" -eq 0 ] + local phase + phase=$(echo "$output" | jq -r '.phase') + [ "$phase" = "plan" ] +} + +@test "progress: reports error in json when events file missing" { + run "$LIB_DIR/archeflow-progress.sh" missing-run --json + # JSON mode returns the JSON even on error + local error + error=$(echo "$output" | jq -r '.error // empty') + [[ "$error" == *"not found"* ]] +} + +@test "progress: rejects unknown flags" { + run "$LIB_DIR/archeflow-progress.sh" test-run --invalid + [ "$status" -eq 1 ] + [[ "$output" == *"Unknown flag"* ]] +} diff --git a/tests/archeflow-report.bats b/tests/archeflow-report.bats new file mode 100644 index 0000000..4e1606d --- /dev/null +++ b/tests/archeflow-report.bats @@ -0,0 +1,80 @@ +# Tests for archeflow-report.sh — Markdown process report generation from JSONL events. +# +# Validates: report output format, summary mode, missing file handling, jq dependency check. + +setup() { + load test_helper + _common_setup + + # Create a standard events file used by multiple tests + mkdir -p .archeflow/events + cat > "$BATS_TEST_TMPDIR/events.jsonl" <<'EVENTS' +{"ts":"2026-04-03T10:00:00Z","run_id":"test-run","seq":1,"parent":[],"type":"run.start","phase":"plan","agent":null,"data":{"task":"Write unit tests","workflow":"standard","team":"default"}} +{"ts":"2026-04-03T10:01:00Z","run_id":"test-run","seq":2,"parent":[1],"type":"agent.complete","phase":"plan","agent":"creator","data":{"archetype":"creator","duration_ms":60000,"tokens":1500,"summary":"Designed test structure"}} +{"ts":"2026-04-03T10:02:00Z","run_id":"test-run","seq":3,"parent":[2],"type":"phase.transition","phase":"do","agent":null,"data":{"from":"plan","to":"do"}} +{"ts":"2026-04-03T10:05:00Z","run_id":"test-run","seq":4,"parent":[3],"type":"agent.complete","phase":"do","agent":"maker","data":{"archetype":"maker","duration_ms":180000,"tokens":3000,"summary":"Implemented tests"}} +{"ts":"2026-04-03T10:06:00Z","run_id":"test-run","seq":5,"parent":[4],"type":"phase.transition","phase":"check","agent":null,"data":{"from":"do","to":"check"}} +{"ts":"2026-04-03T10:07:00Z","run_id":"test-run","seq":6,"parent":[5],"type":"review.verdict","phase":"check","agent":"guardian","data":{"archetype":"guardian","verdict":"approved","findings":[]}} +{"ts":"2026-04-03T10:08:00Z","run_id":"test-run","seq":7,"parent":[6],"type":"run.complete","phase":"act","agent":null,"data":{"status":"completed","cycles":1,"agents_total":3,"fixes_total":0,"duration_ms":480000}} +EVENTS +} + +@test "report: exits 1 with usage when called with no args" { + run "$LIB_DIR/archeflow-report.sh" + [ "$status" -eq 1 ] + [[ "$output" == *"Usage"* ]] +} + +@test "report: exits 1 when events file not found" { + run "$LIB_DIR/archeflow-report.sh" nonexistent.jsonl + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] +} + +@test "report: full mode produces markdown with header and overview" { + run "$LIB_DIR/archeflow-report.sh" "$BATS_TEST_TMPDIR/events.jsonl" + [ "$status" -eq 0 ] + [[ "$output" == *"# Process Report: Write unit tests"* ]] + [[ "$output" == *"test-run"* ]] + [[ "$output" == *"Overview"* ]] + [[ "$output" == *"Status"* ]] + [[ "$output" == *"completed"* ]] +} + +@test "report: full mode includes phase sections" { + run "$LIB_DIR/archeflow-report.sh" "$BATS_TEST_TMPDIR/events.jsonl" + [ "$status" -eq 0 ] + [[ "$output" == *"PLAN"* ]] + [[ "$output" == *"DO"* ]] + [[ "$output" == *"CHECK"* ]] +} + +@test "report: summary mode outputs one-line summary" { + run "$LIB_DIR/archeflow-report.sh" "$BATS_TEST_TMPDIR/events.jsonl" --summary + [ "$status" -eq 0 ] + # Should be a single logical line with key stats + [[ "$output" == *"[completed]"* ]] + [[ "$output" == *"Write unit tests"* ]] + [[ "$output" == *"1 cycles"* ]] + [[ "$output" == *"test-run"* ]] +} + +@test "report: --output writes to file instead of stdout" { + run "$LIB_DIR/archeflow-report.sh" "$BATS_TEST_TMPDIR/events.jsonl" --output "$BATS_TEST_TMPDIR/report.md" + [ "$status" -eq 0 ] + [ -f "$BATS_TEST_TMPDIR/report.md" ] + local content + content=$(cat "$BATS_TEST_TMPDIR/report.md") + [[ "$content" == *"# Process Report"* ]] +} + +@test "report: summary for in-progress run shows [in-progress]" { + # Events file without run.complete + cat > "$BATS_TEST_TMPDIR/in-progress.jsonl" <<'EVENTS' +{"ts":"2026-04-03T10:00:00Z","run_id":"wip-run","seq":1,"parent":[],"type":"run.start","phase":"plan","agent":null,"data":{"task":"WIP task","workflow":"fast","team":"default"}} +EVENTS + run "$LIB_DIR/archeflow-report.sh" "$BATS_TEST_TMPDIR/in-progress.jsonl" --summary + [ "$status" -eq 0 ] + [[ "$output" == *"[in-progress]"* ]] + [[ "$output" == *"WIP task"* ]] +} diff --git a/tests/archeflow-review.bats b/tests/archeflow-review.bats new file mode 100644 index 0000000..4315611 --- /dev/null +++ b/tests/archeflow-review.bats @@ -0,0 +1,82 @@ +# Tests for archeflow-review.sh — git diff extraction for code review. +# +# Validates: argument parsing, diff modes, stats output, empty diff handling. + +setup() { + load test_helper + _common_setup +} + +teardown() { + _common_teardown +} + +@test "review: --help shows usage" { + run "$LIB_DIR/archeflow-review.sh" --help + [ "$status" -eq 0 ] + [[ "$output" == *"Usage"* ]] + [[ "$output" == *"--branch"* ]] + [[ "$output" == *"--commit"* ]] +} + +@test "review: exits 1 when no changes to review" { + run "$LIB_DIR/archeflow-review.sh" + [ "$status" -eq 1 ] + [[ "$output" == *"No changes"* ]] +} + +@test "review: shows diff for uncommitted changes" { + echo "new content" > testfile.txt + git add testfile.txt + run "$LIB_DIR/archeflow-review.sh" + [ "$status" -eq 0 ] + [[ "$output" == *"testfile.txt"* ]] +} + +@test "review: --stat-only prints stats without diff content" { + echo "stat content" > statfile.txt + git add statfile.txt + run "$LIB_DIR/archeflow-review.sh" --stat-only + [ "$status" -eq 0 ] + # stderr has stats, stdout should be empty (no diff) + # But run captures both, so just check it ran ok + [[ "$output" == *"Review Stats"* ]] +} + +@test "review: --branch fails for nonexistent branch" { + run "$LIB_DIR/archeflow-review.sh" --branch nonexistent-branch-xyz + [ "$status" -ne 0 ] + [[ "$output" == *"not found"* ]] +} + +@test "review: rejects unknown arguments" { + run "$LIB_DIR/archeflow-review.sh" --unknown + [ "$status" -ne 0 ] + [[ "$output" == *"Unknown argument"* ]] +} + +@test "review: --branch shows diff against base" { + # Create a feature branch with changes + git checkout -b feat/test-review --quiet + echo "feature" > feature.txt + git add feature.txt + git commit -m "feat: add feature" --quiet + git checkout main --quiet + + run "$LIB_DIR/archeflow-review.sh" --branch feat/test-review + [ "$status" -eq 0 ] + [[ "$output" == *"feature.txt"* ]] +} + +@test "review: --commit shows diff for commit range" { + echo "first" > first.txt + git add first.txt + git commit -m "first" --quiet + echo "second" > second.txt + git add second.txt + git commit -m "second" --quiet + + run "$LIB_DIR/archeflow-review.sh" --commit HEAD~1..HEAD + [ "$status" -eq 0 ] + [[ "$output" == *"second.txt"* ]] +} diff --git a/tests/archeflow-rollback.bats b/tests/archeflow-rollback.bats new file mode 100644 index 0000000..66d612c --- /dev/null +++ b/tests/archeflow-rollback.bats @@ -0,0 +1,58 @@ +# Tests for archeflow-rollback.sh — post-merge test and phase rollback. +# +# Validates: argument parsing, mutual exclusivity, phase validation, test-cmd config reading. + +setup() { + load test_helper + _common_setup +} + +teardown() { + _common_teardown +} + +@test "rollback: exits with error when called with no args" { + run "$LIB_DIR/archeflow-rollback.sh" + [ "$status" -ne 0 ] +} + +@test "rollback: rejects mutually exclusive --to and --test-cmd" { + run "$LIB_DIR/archeflow-rollback.sh" test-run --to plan --test-cmd "true" + [ "$status" -eq 2 ] + [[ "$output" == *"mutually exclusive"* ]] +} + +@test "rollback: rejects invalid phase names" { + run "$LIB_DIR/archeflow-rollback.sh" test-run --to invalid-phase + [ "$status" -eq 2 ] + [[ "$output" == *"Invalid phase"* ]] +} + +@test "rollback: accepts valid phase names (plan, do, check)" { + # This will fail because no git branch exists, but should NOT fail on phase validation + run "$LIB_DIR/archeflow-rollback.sh" test-run --to plan + # Should fail later (archeflow-git.sh rollback) not on phase validation + [[ "$output" != *"Invalid phase"* ]] +} + +@test "rollback: exits 2 when no test command available" { + run "$LIB_DIR/archeflow-rollback.sh" test-run + [ "$status" -eq 2 ] + [[ "$output" == *"No test command"* ]] +} + +@test "rollback: reads test_command from config.yaml" { + mkdir -p .archeflow + echo 'test_command: "echo ok"' > .archeflow/config.yaml + # HEAD won't have archeflow in its message, but the script just warns and proceeds + run "$LIB_DIR/archeflow-rollback.sh" test-run + # It should pick up the command and try to run it (test should pass -> exit 0) + [ "$status" -eq 0 ] + [[ "$output" == *"Tests passed"* ]] +} + +@test "rollback: rejects unknown options" { + run "$LIB_DIR/archeflow-rollback.sh" test-run --unknown-flag + [ "$status" -eq 2 ] + [[ "$output" == *"Unknown option"* ]] +} diff --git a/tests/archeflow-score.bats b/tests/archeflow-score.bats new file mode 100644 index 0000000..960c6a1 --- /dev/null +++ b/tests/archeflow-score.bats @@ -0,0 +1,105 @@ +# Tests for archeflow-score.sh — archetype effectiveness scoring. +# +# Validates: score extraction from events, report generation, input validation. + +setup() { + load test_helper + _common_setup + + # Create a complete run events file with review data + mkdir -p .archeflow/events .archeflow/memory + cat > "$BATS_TEST_TMPDIR/scored-events.jsonl" <<'EVENTS' +{"ts":"2026-04-03T10:00:00Z","run_id":"score-run","seq":1,"parent":[],"type":"run.start","phase":"plan","agent":null,"data":{"task":"Score test"}} +{"ts":"2026-04-03T10:01:00Z","run_id":"score-run","seq":2,"parent":[1],"type":"agent.complete","phase":"plan","agent":"creator","data":{"archetype":"creator","duration_ms":60000,"tokens":1500,"estimated_cost_usd":0.02}} +{"ts":"2026-04-03T10:02:00Z","run_id":"score-run","seq":3,"parent":[2],"type":"agent.complete","phase":"do","agent":"maker","data":{"archetype":"maker","duration_ms":120000,"tokens":3000,"estimated_cost_usd":0.05}} +{"ts":"2026-04-03T10:03:00Z","run_id":"score-run","seq":4,"parent":[3],"type":"review.verdict","phase":"check","agent":"guardian","data":{"archetype":"guardian","verdict":"needs_changes","findings":[{"severity":"warning","description":"Missing validation","fix_required":true},{"severity":"info","description":"Consider logging","fix_required":false}]}} +{"ts":"2026-04-03T10:03:30Z","run_id":"score-run","seq":5,"parent":[3],"type":"review.verdict","phase":"check","agent":"sage","data":{"archetype":"sage","verdict":"approved","findings":[]}} +{"ts":"2026-04-03T10:04:00Z","run_id":"score-run","seq":6,"parent":[4],"type":"fix.applied","phase":"act","agent":null,"data":{"source":"guardian","finding":"Missing validation"}} +{"ts":"2026-04-03T10:05:00Z","run_id":"score-run","seq":7,"parent":[6],"type":"cycle.boundary","phase":"act","agent":null,"data":{"cycle":1,"max_cycles":3,"met":true,"next_action":"merge"}} +{"ts":"2026-04-03T10:06:00Z","run_id":"score-run","seq":8,"parent":[7],"type":"run.complete","phase":"act","agent":null,"data":{"status":"completed","cycles":1,"agents_total":4,"fixes_total":1}} +EVENTS +} + +@test "score: exits 1 with usage when called with no args" { + run "$LIB_DIR/archeflow-score.sh" + [ "$status" -eq 1 ] + [[ "$output" == *"Usage"* ]] +} + +@test "score: exits 1 for unknown command" { + run "$LIB_DIR/archeflow-score.sh" nonexistent + [ "$status" -eq 1 ] + [[ "$output" == *"Unknown command"* ]] +} + +@test "score extract: exits 1 when events file not found" { + run "$LIB_DIR/archeflow-score.sh" extract nonexistent.jsonl + [ "$status" -eq 1 ] + [[ "$output" == *"not found"* ]] +} + +@test "score extract: exits 1 for incomplete run (no run.complete)" { + cat > "$BATS_TEST_TMPDIR/incomplete.jsonl" <<'EVENTS' +{"ts":"2026-04-03T10:00:00Z","run_id":"incomplete","seq":1,"parent":[],"type":"run.start","phase":"plan","agent":null,"data":{"task":"Incomplete"}} +EVENTS + run "$LIB_DIR/archeflow-score.sh" extract "$BATS_TEST_TMPDIR/incomplete.jsonl" + [ "$status" -eq 1 ] + [[ "$output" == *"run.complete"* ]] +} + +@test "score extract: creates effectiveness.jsonl with archetype scores" { + run "$LIB_DIR/archeflow-score.sh" extract "$BATS_TEST_TMPDIR/scored-events.jsonl" + [ "$status" -eq 0 ] + [ -f ".archeflow/memory/effectiveness.jsonl" ] + + # Should have scores for guardian and sage (the reviewers) + local guardian_score + guardian_score=$(grep '"guardian"' ".archeflow/memory/effectiveness.jsonl" | head -1) + [ -n "$guardian_score" ] + + # Verify JSONL is valid + while IFS= read -r line; do + echo "$line" | jq empty + done < ".archeflow/memory/effectiveness.jsonl" +} + +@test "score extract: guardian has correct finding counts" { + "$LIB_DIR/archeflow-score.sh" extract "$BATS_TEST_TMPDIR/scored-events.jsonl" 2>/dev/null + local guardian + guardian=$(grep '"guardian"' ".archeflow/memory/effectiveness.jsonl" | head -1) + local total_findings + total_findings=$(echo "$guardian" | jq '.findings_total') + [ "$total_findings" -eq 2 ] + local useful_findings + useful_findings=$(echo "$guardian" | jq '.findings_useful') + [ "$useful_findings" -eq 1 ] + local fixes + fixes=$(echo "$guardian" | jq '.fixes_applied') + [ "$fixes" -eq 1 ] +} + +@test "score extract: composite score is between 0 and 1" { + "$LIB_DIR/archeflow-score.sh" extract "$BATS_TEST_TMPDIR/scored-events.jsonl" 2>/dev/null + while IFS= read -r line; do + local score + score=$(echo "$line" | jq '.composite_score') + # score >= 0 and score <= 1 + [ "$(echo "$score >= 0" | bc)" -eq 1 ] + [ "$(echo "$score <= 1" | bc)" -eq 1 ] + done < ".archeflow/memory/effectiveness.jsonl" +} + +@test "score report: exits 1 when no effectiveness data" { + run "$LIB_DIR/archeflow-score.sh" report + [ "$status" -eq 1 ] + [[ "$output" == *"No effectiveness data"* ]] +} + +@test "score report: outputs markdown table with archetype data" { + "$LIB_DIR/archeflow-score.sh" extract "$BATS_TEST_TMPDIR/scored-events.jsonl" 2>/dev/null + run "$LIB_DIR/archeflow-score.sh" report + [ "$status" -eq 0 ] + [[ "$output" == *"Archetype Effectiveness Report"* ]] + [[ "$output" == *"Archetype"* ]] + [[ "$output" == *"guardian"* ]] +} diff --git a/tests/test_helper.bash b/tests/test_helper.bash new file mode 100644 index 0000000..8525ce2 --- /dev/null +++ b/tests/test_helper.bash @@ -0,0 +1,40 @@ +# test_helper.bash — Shared setup/teardown for ArcheFlow bats tests. +# +# Usage in .bats files: +# setup() { load test_helper; _common_setup; } +# teardown() { _common_teardown; } +# +# Provides: +# - BATS_TEST_TMPDIR: unique temp directory per test +# - Mock .archeflow/ structure via a git repo +# - LIB_DIR: path to the lib/ scripts under test + +LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)" + +_common_setup() { + # Create a unique temp directory for this test + BATS_TEST_TMPDIR="$(mktemp -d)" + export BATS_TEST_TMPDIR + + # Work inside the temp dir so scripts create .archeflow/ there + cd "$BATS_TEST_TMPDIR" + + # Initialize a minimal git repo (many scripts need it) + git init --quiet + git config user.email "test@test.com" + git config user.name "Test User" + # Disable commit signing in tests (global config may have it enabled) + git config commit.gpgsign false + git config tag.gpgsign false + + # Create an initial commit so HEAD exists + echo "init" > README.md + git add README.md + git commit -m "init" --quiet +} + +_common_teardown() { + # Return to a safe directory before cleanup + cd /tmp + rm -rf "$BATS_TEST_TMPDIR" +} From 607a53f1bf8ab4658033dd47af682fcdb82786d3 Mon Sep 17 00:00:00 2001 From: Christian Nennemann Date: Mon, 6 Apr 2026 21:33:36 +0200 Subject: [PATCH 2/2] feat: add decision.point event type, decision logger, and run replay script - archeflow-decision.sh: convenience wrapper for logging PDCA decision points - archeflow-replay.sh: timeline view and weighted what-if replay for recorded runs - archeflow-event.sh: add decision.point usage example - archeflow-dag.sh: render decision.point events in DAG output --- lib/archeflow-dag.sh | 5 +- lib/archeflow-decision.sh | 48 ++++++++ lib/archeflow-event.sh | 3 + lib/archeflow-replay.sh | 228 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 283 insertions(+), 1 deletion(-) create mode 100755 lib/archeflow-decision.sh create mode 100755 lib/archeflow-replay.sh diff --git a/lib/archeflow-dag.sh b/lib/archeflow-dag.sh index d1212de..78d0634 100755 --- a/lib/archeflow-dag.sh +++ b/lib/archeflow-dag.sh @@ -87,6 +87,9 @@ EVENTS_PARSED=$(jq -r ' elif .type == "agent.complete" then (.data.archetype // .agent // "unknown") + " (" + .phase + ")" + (if (.data.tokens // 0) > 0 then " [" + (.data.tokens | tostring) + " tok]" else "" end) + elif .type == "decision.point" then + (.data.archetype // .agent // "?") + " → " + (.data.decision // "?") + + " (conf " + ((.data.confidence // 0) | tostring) + ")" elif .type == "decision" then "decision: " + (.data.what // "unknown") + " → " + (.data.chosen // "unknown") elif .type == "phase.transition" then @@ -209,7 +212,7 @@ render_node() { local colored_label case "$type" in phase.transition) colored_label="${C_TRANS}${label}${C_RESET}" ;; - decision) colored_label="${C_DECISION}${label}${C_RESET}" ;; + decision|decision.point) colored_label="${C_DECISION}${label}${C_RESET}" ;; review.verdict) colored_label="${C_VERDICT}${label}${C_RESET}" ;; *) colored_label="${pc}${label}${C_RESET}" ;; esac diff --git a/lib/archeflow-decision.sh b/lib/archeflow-decision.sh new file mode 100755 index 0000000..1190f6e --- /dev/null +++ b/lib/archeflow-decision.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# archeflow-decision.sh — Log a PDCA decision point for run replay / effectiveness analysis. +# +# Appends a decision.point event to .archeflow/events/.jsonl with: +# phase, archetype (agent + data.archetype), input, decision, confidence, ts (via event layer) +# +# Usage: +# ./lib/archeflow-decision.sh '' '' [parent_seq] +# +# Examples: +# ./lib/archeflow-decision.sh 2026-04-06-auth check guardian \ +# 'diff + proposal risks' 'needs_changes' 0.82 7 +# ./lib/archeflow-decision.sh 2026-04-06-auth act "" 'route findings' 'send_to_maker' 0.9 +# +# confidence: 0.0–1.0 (orchestrator-estimated certainty in the recorded choice) +# +# Requires: jq (via archeflow-event.sh) + +set -euo pipefail + +LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [[ $# -lt 6 ]]; then + echo "Usage: $0 '' '' [parent_seq]" >&2 + exit 1 +fi + +RUN_ID="$1" +PHASE="$2" +ARCH="$3" +INPUT="$4" +DECISION="$5" +CONF_RAW="$6" +PARENT="${7:-}" + +if ! [[ "$CONF_RAW" =~ ^[0-9]*\.?[0-9]+$ ]]; then + echo "Error: confidence must be a number (e.g. 0.85)" >&2 + exit 1 +fi + +DATA=$(jq -cn \ + --arg a "$ARCH" \ + --arg i "$INPUT" \ + --arg d "$DECISION" \ + --argjson c "$CONF_RAW" \ + '{archetype:$a, input:$i, decision:$d, confidence:$c}') + +exec "$LIB_DIR/archeflow-event.sh" "$RUN_ID" decision.point "$PHASE" "$ARCH" "$DATA" "$PARENT" diff --git a/lib/archeflow-event.sh b/lib/archeflow-event.sh index 5a3859b..ab0a728 100755 --- a/lib/archeflow-event.sh +++ b/lib/archeflow-event.sh @@ -8,6 +8,9 @@ # ./lib/archeflow-event.sh 2026-04-03-der-huster agent.complete plan creator '{"duration_ms":167522}' 2 # ./lib/archeflow-event.sh 2026-04-03-der-huster phase.transition do "" '{"from":"plan","to":"do"}' 3,4 # ./lib/archeflow-event.sh 2026-04-03-der-huster fix.applied act "" '{"source":"guardian"}' 8 +# ./lib/archeflow-event.sh 2026-04-03-der-huster decision.point check guardian \ +# '{"archetype":"guardian","input":"diff","decision":"needs_changes","confidence":0.85}' 7 +# # Or use: ./lib/archeflow-decision.sh '' '' [parent] # # Parent seqs: comma-separated seq numbers of causal parent events (DAG). # "2" → single parent [2] diff --git a/lib/archeflow-replay.sh b/lib/archeflow-replay.sh new file mode 100755 index 0000000..a621d11 --- /dev/null +++ b/lib/archeflow-replay.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# archeflow-replay.sh — Inspect recorded runs: decision timeline and weighted what-if replay. +# +# Usage: +# archeflow-replay.sh timeline +# archeflow-replay.sh whatif [--weights arch=w,arch2=w2] [--threshold 0.5] [--json] +# archeflow-replay.sh compare [--weights ...] [--threshold ...] [--json] +# +# Events file: .archeflow/events/.jsonl (relative to current working directory) +# +# whatif / compare: +# - Loads check-phase review.verdict events (last verdict per archetype). +# - Original gate (strict): BLOCK if any reviewer is not approved. +# - Replay gate (weighted): BLOCK if sum(weight * strict) / sum(weight) >= threshold, +# where strict=1 for non-approved verdicts, else 0. Default weight per archetype is 1.0. +# +# Requires: jq + +set -euo pipefail + +if [[ $# -lt 2 ]]; then + echo "Usage: $0 {timeline|whatif|compare} [options]" >&2 + echo "" >&2 + echo " timeline Decision timeline (decision.point + review.verdict)" >&2 + echo " whatif [--weights k=v,...] [--threshold 0.5] [--json]" >&2 + echo " compare (timeline + whatif summary)" >&2 + exit 1 +fi + +COMMAND="$1" +RUN_ID="$2" +shift 2 + +if ! command -v jq &>/dev/null; then + echo "Error: jq is required." >&2 + exit 1 +fi + +EVENT_FILE=".archeflow/events/${RUN_ID}.jsonl" + +resolve_event_file() { + if [[ ! -f "$EVENT_FILE" ]]; then + echo "Error: event file not found: $EVENT_FILE" >&2 + exit 1 + fi +} + +cmd_timeline() { + resolve_event_file + echo "## Decision timeline — run_id=${RUN_ID}" + echo "" + local cnt + cnt=$(jq -s '[.[] | select(.type == "decision.point")] | length' "$EVENT_FILE") + if [[ "$cnt" -gt 0 ]]; then + echo "### decision.point (${cnt})" + jq -r 'select(.type == "decision.point") + | "- \(.ts) [\(.phase)] \(.data.archetype // .agent // "?") \(.data.decision) conf=\(.data.confidence // "n/a") input=\(.data.input // "")"' \ + "$EVENT_FILE" + echo "" + else + echo "### decision.point" + echo "(none — emit with ./lib/archeflow-decision.sh during the run)" + echo "" + fi + + echo "### review.verdict (check phase)" + if jq -e -s '[.[] | select(.type == "review.verdict" and .phase == "check")] | length > 0' "$EVENT_FILE" >/dev/null 2>&1; then + jq -r 'select(.type == "review.verdict" and .phase == "check") + | "- \(.ts) \(.data.archetype // .agent // "?") verdict=\(.data.verdict) findings=\((.data.findings // []) | length)"' \ + "$EVENT_FILE" + else + echo "(none)" + fi + echo "" +} + +parse_weights_to_json() { + local raw="${1:-}" + local obj='{}' + if [[ -z "$raw" ]]; then + echo '{}' + return + fi + IFS=',' read -ra pairs <<< "$raw" + for pair in "${pairs[@]}"; do + [[ -z "$pair" ]] && continue + local k="${pair%%=*}" + local v="${pair#*=}" + k=$(echo "$k" | tr '[:upper:]' '[:lower:]' | xargs) + v=$(echo "$v" | xargs) + if [[ -z "$k" || "$k" == "$pair" ]]; then + echo "Error: invalid weight entry (use arch=1.5): $pair" >&2 + exit 1 + fi + obj=$(echo "$obj" | jq --arg k "$k" --argjson v "$v" '. + {($k): $v}') + done + echo "$obj" +} + +cmd_whatif() { + local weights_str="" + local threshold="0.5" + local json_out="false" + while [[ $# -gt 0 ]]; do + case "$1" in + --weights) + weights_str="$2" + shift 2 + ;; + --threshold) + threshold="$2" + shift 2 + ;; + --json) + json_out="true" + shift + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac + done + + resolve_event_file + local weights_json + weights_json="$(parse_weights_to_json "$weights_str")" + + local result + result=$(jq -s --argjson weights "$weights_json" --argjson thr "$threshold" --arg run_id "$RUN_ID" ' + def strict($v): + if $v == null then 1 + else ($v | ascii_downcase) as $lv + | if ($lv == "approved" or $lv == "approve") then 0 else 1 end + end; + + def norm_key: ascii_downcase; + + ([.[] | select(.type == "review.verdict" and .phase == "check")] + | sort_by(.seq) + | reduce .[] as $e ({}; . + { (($e.data.archetype // $e.agent // "unknown") | norm_key): $e }) + ) as $last | + + ($last | keys) as $keys | + if ($keys | length) == 0 then + { + run_id: $run_id, + error: "no check-phase review.verdict events; nothing to simulate" + } + else + [ $keys[] as $k | $last[$k] as $ev | + ($weights[($k | norm_key)] // 1.0) as $w + | strict($ev.data.verdict) as $s + | { + archetype: ($ev.data.archetype // $ev.agent // $k), + verdict: ($ev.data.verdict // "unknown"), + weight: $w, + strict: $s, + weighted_contrib: ($w * $s) + } + ] as $rows | + ($rows | map(.weighted_contrib) | add) as $num | + ($rows | map(.weight) | add) as $den | + (if $den > 0 then ($num / $den) else 0 end) as $ratio | + (if ($rows | map(.strict) | max) == 1 then "BLOCK" else "SHIP" end) as $strict_out | + (if $ratio >= $thr then "BLOCK" else "SHIP" end) as $replay_out | + { + run_id: $run_id, + threshold: $thr, + weights_used: $weights, + strict_any_veto: { + outcome: $strict_out, + description: "BLOCK if any reviewer verdict is not approved" + }, + weighted_replay: { + weighted_strictness: ($ratio * 1000 | round / 1000), + outcome: $replay_out, + description: ("BLOCK if weighted strictness >= " + ($thr | tostring)) + }, + reviewers: $rows + } + end + ' "$EVENT_FILE") + + if [[ "$json_out" == "true" ]]; then + echo "$result" + else + echo "$result" | jq -r ' + if .error then "Error: \(.error)" else + "# What-if replay — run_id=\(.run_id)\n", + "", + "## Outcomes", + "| Model | Result |", + "|-------|--------|", + "| Original (any non-approve → BLOCK) | \(.strict_any_veto.outcome) |", + "| Weighted replay (threshold=\(.threshold)) | \(.weighted_replay.outcome) |", + "", + "## Weighted strictness", + "\(.weighted_replay.weighted_strictness) (0 = all approved, 1 = all blocking)", + "", + "## Per reviewer", + "| Archetype | Verdict | Weight | Strict | w×strict |", + "|-----------|---------|--------|--------|----------|", + (.reviewers[] | "| \(.archetype) | \(.verdict) | \(.weight) | \(.strict) | \(.weighted_contrib) |"), + "", + (if (.weights_used | length) > 0 then + "## Custom weights applied\n" + (.weights_used | to_entries | map("- \(.key): \(.value)") | join("\n")) + "\n" + else empty end) + end + ' + fi +} + +cmd_compare() { + cmd_timeline + echo "" + cmd_whatif "$@" +} + +case "$COMMAND" in + timeline) cmd_timeline ;; + whatif) cmd_whatif "$@" ;; + compare) cmd_compare "$@" ;; + *) + echo "Unknown command: $COMMAND" >&2 + exit 1 + ;; +esac