diff --git a/claimlint/test_claimlint.sh b/claimlint/test_claimlint.sh new file mode 100755 index 0000000..0326f85 --- /dev/null +++ b/claimlint/test_claimlint.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# Tests for claimlint. Ground truth = synthesized Claude-Code-shaped transcripts pairing the +# SAME completion claim with a session that does vs. doesn't support it — plus the FP guards +# (future tense, conditional) that must NOT trip, and --root filesystem verification. +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" +CL="$HERE/claimlint"; chmod +x "$CL" +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +PASS=0; FAIL=0 +ok(){ PASS=$((PASS+1)); echo "PASS: $1"; } +bad(){ FAIL=$((FAIL+1)); echo "FAIL: $1 -- $2"; } + +mk(){ local out="$1"; shift; W="$W" OUT="$out" python3 -c " +import json,os +rows=[] +$* +open(os.environ['OUT'],'w').write('\n'.join(json.dumps(r) for r in rows)) +"; } +# helpers usable inside mk bodies +A_TEXT(){ echo "rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'text','text':\"$1\"}]}})"; } +A_TOOL(){ echo "rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':'$1','name':'$2','input':$3}]}})"; } +RESULT(){ echo "rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':'$1','is_error':$2,'content':$3}]}})"; } + +# T1: HONEST — claim backed by a real passing test run + a real Write -> exit 0 +mk "$W/honest.jsonl" " +$(A_TOOL w1 Write "{'file_path':'src/auth.py','content':'def login(): ...'}") +$(RESULT w1 False "'File created successfully'") +$(A_TOOL b1 Bash "{'command':'pytest -q'}") +$(RESULT b1 False "'3 passed in 0.4s'") +$(A_TEXT 'All tests pass and I created src/auth.py. Done.') +" +out=$("$CL" "$W/honest.jsonl"); rc=$? +{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T1 honest (test+file backed) = exit 0" || bad T1 "$out" + +# T2: LYING — claims pass + created, neither happened -> both findings, exit 1 +mk "$W/lying.jsonl" " +$(A_TOOL r1 Read "{'file_path':'main.py'}") +$(RESULT r1 False "'code'") +$(A_TEXT 'All tests pass and I created src/auth.py. Done.') +" +out=$("$CL" "$W/lying.jsonl"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "TEST-CLAIM" && echo "$out" | grep -q "FILE-CLAIM"; } && ok "T2 lying (no test, no write) flags both + exit 1" || bad T2 "$out" + +# T3: FP — FUTURE tense ('I will create / run the tests') must NOT flag +mk "$W/future.jsonl" " +$(A_TOOL r1 Read "{'file_path':'main.py'}") +$(RESULT r1 False "'code'") +$(A_TEXT 'Next I will create src/auth.py and then the tests should pass.') +" +out=$("$CL" "$W/future.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T3 future-tense plan NOT flagged" || bad T3 "$out" + +# T4: FP — CONDITIONAL ('if the tests pass') must NOT flag +mk "$W/cond.jsonl" " +$(A_TOOL r1 Read "{'file_path':'main.py'}") +$(RESULT r1 False "'code'") +$(A_TEXT 'If the tests pass we are done; run them to confirm.') +" +out=$("$CL" "$W/cond.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T4 conditional claim NOT flagged" || bad T4 "$out" + +# T5: TEST-CLAIM — claims pass but the test run ERRORED -> flagged +mk "$W/testfail.jsonl" " +$(A_TOOL b1 Bash "{'command':'pytest -q'}") +$(RESULT b1 True "'2 failed, 1 passed'") +$(A_TEXT 'All tests pass now.') +" +out=$("$CL" "$W/testfail.jsonl"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "TEST-CLAIM"; } && ok "T5 claim-pass but test failed flagged" || bad T5 "$out" + +# T6: FILE-CLAIM basename match — claim 'auth.py', Write was 'src/auth.py' -> NOT flagged +mk "$W/basename.jsonl" " +$(A_TOOL w1 Write "{'file_path':'src/auth.py','content':'x'}") +$(RESULT w1 False "'ok'") +$(A_TEXT 'I created auth.py for you.') +" +out=$("$CL" "$W/basename.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T6 file claim matched by basename NOT flagged" || bad T6 "$out" + +# T7: --root — claimed file exists ON DISK though no edit tool call -> NOT flagged +mkdir -p "$W/repo/lib"; echo "x" > "$W/repo/lib/util.py" +mk "$W/disk.jsonl" " +$(A_TOOL r1 Read "{'file_path':'main.py'}") +$(RESULT r1 False "'code'") +$(A_TEXT 'I updated lib/util.py.') +" +out=$("$CL" --root "$W/repo" "$W/disk.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T7 --root: file on disk NOT flagged" || bad T7 "$out" + +# T8: --root — claimed file NOT edited and NOT on disk -> flagged +mk "$W/ghost.jsonl" " +$(A_TOOL r1 Read "{'file_path':'main.py'}") +$(RESULT r1 False "'code'") +$(A_TEXT 'I created lib/ghost.py with the helper.') +" +out=$("$CL" --root "$W/repo" "$W/ghost.jsonl"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "FILE-CLAIM"; } && ok "T8 --root: ghost file flagged" || bad T8 "$out" + +# T9: non-file tokens (e.g., version numbers) NOT treated as file claims +mk "$W/notfile.jsonl" " +$(A_TOOL b1 Bash "{'command':'pytest -q'}") +$(RESULT b1 False "'5 passed'") +$(A_TEXT 'All tests pass. Updated to v2.1 and documented e.g. the flow.') +" +out=$("$CL" "$W/notfile.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T9 version/e.g. not treated as files" || bad T9 "$out" + +# T10: stdin mode +out=$(cat "$W/lying.jsonl" | "$CL"); echo "$out" | grep -q "FILE-CLAIM" && ok "T10 stdin mode" || bad T10 "$out" + +# T11: --json well-formed +j=$("$CL" --json "$W/lying.jsonl"); echo "$j" | python3 -c 'import sys,json;d=json.load(sys.stdin);assert d["clean"] is False and len(d["findings"])>=2' && ok "T11 --json well-formed" || bad T11 "$j" + +# T12: --help +"$CL" --help | grep -q "completion CLAIMS" && ok "T12 --help" || bad T12 "no help" + +# T13: missing file = exit 2 +"$CL" "$W/nope.jsonl" >/dev/null 2>&1; [ "$?" -eq 2 ] && ok "T13 missing file = exit 2" || bad T13 "rc" + +# T14: not-a-transcript = exit 2 +printf 'hello world\n' > "$W/plain.txt"; "$CL" "$W/plain.txt" >/dev/null 2>&1; [ "$?" -eq 2 ] && ok "T14 non-transcript = exit 2" || bad T14 "rc" + +# T15: honest 'Done.' with no specific claim and supporting edit -> exit 0 (no false alarm on bare 'Done') +mk "$W/baredone.jsonl" " +$(A_TOOL w1 Edit "{'file_path':'README.md','old_string':'a','new_string':'b'}") +$(RESULT w1 False "'ok'") +$(A_TEXT 'Done.') +" +out=$("$CL" "$W/baredone.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T15 bare 'Done.' not falsely flagged" || bad T15 "$out" + +# T16: list-shaped tool_result content parsed (passing test in list content) +mk "$W/listtest.jsonl" " +$(A_TOOL b1 Bash "{'command':'go test ./...'}") +rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':'b1','is_error':False,'content':[{'type':'text','text':'ok all 4 passed'}]}]}}) +$(A_TEXT 'All tests pass.') +" +out=$("$CL" "$W/listtest.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T16 list-content passing test supports claim" || bad T16 "$out" + +echo "-----"; echo "PASS=$PASS FAIL=$FAIL"; [ "$FAIL" -eq 0 ] diff --git a/langcheck/test_langcheck.sh b/langcheck/test_langcheck.sh new file mode 100755 index 0000000..1f579df --- /dev/null +++ b/langcheck/test_langcheck.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Tests for langcheck. Ground truth = real text samples (the actual bug shapes: +# CJK in English output, mojibake, runaway repetition) + a real transcript JSONL. +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" +LC="$HERE/langcheck"; chmod +x "$LC" +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +PASS=0; FAIL=0 +ok(){ PASS=$((PASS+1)); echo "PASS: $1"; } +bad(){ FAIL=$((FAIL+1)); echo "FAIL: $1 -- $2"; } + +# T1: clean English -> exit 0, "clean" +printf 'The quick brown fox jumps over the lazy dog.\nThis is a normal paragraph of English output.\n' > "$W/clean.txt" +out=$("$LC" "$W/clean.txt"); rc=$? +{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T1 clean English = exit 0" || bad T1 "$out" + +# T2: English output with a random CJK run (the actual bug) -> SCRIPT-INTRUSION, exit 1 +python3 -c 'open("'"$W/cjk.txt"'","w",encoding="utf-8").write("Here is the function you asked for. 这是一个 it returns the value.\nAll good otherwise.\n")' +out=$("$LC" "$W/cjk.txt"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "SCRIPT-INTRUSION" && echo "$out" | grep -q "CJK"; } && ok "T2 CJK-in-English flagged + exit 1" || bad T2 "$out" + +# T3: a genuinely Chinese document is NOT flagged as intrusion +python3 -c 'open("'"$W/zh.txt"'","w",encoding="utf-8").write("这是一个完全用中文写的文档。它包含许多中文句子和段落。没有英文。\n第二行也是中文内容。\n")' +out=$("$LC" "$W/zh.txt"); rc=$? +{ [ "$rc" -eq 0 ] && ! echo "$out" | grep -q "SCRIPT-INTRUSION"; } && ok "T3 predominantly-Chinese doc not false-flagged" || bad T3 "$out" + +# T4: mojibake replacement char -> MOJIBAKE +python3 -c 'open("'"$W/moji.txt"'","w",encoding="utf-8").write("The price is �25 today and the rest is fine.\n")' +out=$("$LC" "$W/moji.txt") +echo "$out" | grep -q "MOJIBAKE" && ok "T4 mojibake (U+FFFD) flagged" || bad T4 "$out" + +# T5: runaway repetition -> REPETITION +{ echo "Generating the report now."; for i in 1 2 3 4 5; do echo "I will now continue."; done; echo "Done."; } > "$W/rep.txt" +out=$("$LC" "$W/rep.txt") +echo "$out" | grep -q "REPETITION" && echo "$out" | grep -q "5x" && ok "T5 runaway repetition flagged (5x)" || bad T5 "$out" + +# T5b: below threshold (3x default 4) NOT flagged +{ for i in 1 2 3; do echo "ok line"; done; echo "end"; } > "$W/rep2.txt" +out=$("$LC" "$W/rep2.txt"); echo "$out" | grep -q "REPETITION" && bad T5b "3x wrongly flagged" || ok "T5b under-threshold repetition not flagged" + +# T6: emoji / accented Latin in English NOT flagged as foreign-script +python3 -c 'open("'"$W/emoji.txt"'","w",encoding="utf-8").write("Great work \U0001F389 the café is naïve about résumés.\n")' +out=$("$LC" "$W/emoji.txt"); rc=$? +{ [ "$rc" -eq 0 ] && ! echo "$out" | grep -q "SCRIPT-INTRUSION"; } && ok "T6 emoji + accented Latin not flagged" || bad T6 "$out" + +# T7: transcript mode extracts assistant text + flags the CJK in it +python3 -c ' +import json +rows=[ + {"type":"user","message":{"role":"user","content":[{"type":"text","text":"hi"}]}}, + {"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Sure, here is the plan 部署 and the steps."}]}}, +] +open("'"$W/tx.jsonl"'","w").write("\n".join(json.dumps(r) for r in rows)) +' +out=$("$LC" --transcript "$W/tx.jsonl"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "SCRIPT-INTRUSION"; } && ok "T7 transcript mode flags CJK in assistant text" || bad T7 "$out" + +# T8: stdin mode +out=$(printf 'normal english here\n' | "$LC"); echo "$out" | grep -q "clean" && ok "T8 stdin clean" || bad T8 "$out" +out=$(python3 -c 'print("english then 한국어 hangul intrusion")' | "$LC"); echo "$out" | grep -q "Hangul" && ok "T8b stdin flags Hangul intrusion" || bad T8b "$out" + +# T9: --help +"$LC" --help | grep -q "contamination in LLM" && ok "T9 --help" || bad T9 "no help" + +# T10: missing file = exit 2 +"$LC" "$W/nope.txt" >/dev/null 2>&1; [ "$?" -eq 2 ] && ok "T10 missing file = exit 2" || bad T10 "rc" + +echo "-----"; echo "PASS=$PASS FAIL=$FAIL"; [ "$FAIL" -eq 0 ] diff --git a/leaklint/test_leaklint.sh b/leaklint/test_leaklint.sh new file mode 100755 index 0000000..ea2afb5 --- /dev/null +++ b/leaklint/test_leaklint.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Tests for leaklint. Ground truth = real secret SHAPES (vendor key formats, PEM headers, +# KEY= assignments) + placeholder/env-ref negatives + a transcript with a key in a +# tool_use input. The DEFINING test asserts the tool never echoes a secret's value. +# NOTE: all "secrets" below are obviously-fake random strings generated at test time — no +# real credential is ever hardcoded here. +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" +LL="$HERE/leaklint"; chmod +x "$LL" +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +PASS=0; FAIL=0 +ok(){ PASS=$((PASS+1)); echo "PASS: $1"; } +bad(){ FAIL=$((FAIL+1)); echo "FAIL: $1 -- $2"; } + +# fake-but-shaped secrets (random tails so they trip format/entropy but are not real) +ANT="sk-ant-api03-$(python3 -c 'import random,string;print("".join(random.choice(string.ascii_letters+string.digits+"_-") for _ in range(90)))')" +GH="ghp_$(python3 -c 'import random,string;print("".join(random.choice(string.ascii_letters+string.digits) for _ in range(36)))')" +AWS="AKIA$(python3 -c 'import random,string;print("".join(random.choice(string.ascii_uppercase+string.digits) for _ in range(16)))')" +RAND="$(python3 -c 'import random,string;print("".join(random.choice(string.ascii_letters+string.digits) for _ in range(40)))')" + +# T1: clean prose -> exit 0, "clean" +printf 'Here is the function you asked for. It reads the key from the environment.\nAll good.\n' > "$W/clean.txt" +out=$("$LL" "$W/clean.txt"); rc=$? +{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T1 clean prose = exit 0" || bad T1 "$out" + +# T2: Anthropic key -> PROVIDER-KEY, exit 1 +printf 'Sure, use this key: %s in your config.\n' "$ANT" > "$W/ant.txt" +out=$("$LL" "$W/ant.txt"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "PROVIDER-KEY" && echo "$out" | grep -q "Anthropic"; } && ok "T2 Anthropic key flagged + exit 1" || bad T2 "$out" + +# T3 (DEFINING): the secret VALUE never appears in stdout, in any mode +out=$("$LL" "$W/ant.txt"); j=$("$LL" --json "$W/ant.txt") +if echo "$out$j" | grep -qF "$ANT"; then bad T3 "secret value LEAKED into output"; else ok "T3 DEFINING: value never printed (text+json)"; fi +# and the json must carry no "value" key +echo "$j" | grep -q '"value"' && bad T3b "json has a value key" || ok "T3b json has no value key" + +# T4: GitHub token + AWS key both flagged +printf 'token=%s and aws=%s\n' "$GH" "$AWS" > "$W/multi.txt" +out=$("$LL" "$W/multi.txt"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "GitHub token" && echo "$out" | grep -q "AWS access key"; } && ok "T4 GitHub + AWS flagged" || bad T4 "$out" + +# T5: PEM private-key header flagged, body bytes never printed +printf -- '-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAAB\n-----END OPENSSH PRIVATE KEY-----\n' > "$W/pem.txt" +out=$("$LL" "$W/pem.txt") +{ echo "$out" | grep -q "PRIVATE-KEY" && ! echo "$out" | grep -q "b3BlbnNz"; } && ok "T5 PEM header flagged, body not printed" || bad T5 "$out" + +# T6: high-entropy KEY= assignment flagged (ASSIGNED-SECRET) +printf 'OPENAI_API_KEY = "%s"\n' "$RAND" > "$W/assign.txt" +out=$("$LL" "$W/assign.txt") +echo "$out" | grep -q "ASSIGNED-SECRET" && ok "T6 KEY= flagged" || bad T6 "$out" + +# T7: placeholder NOT flagged +printf 'OPENAI_API_KEY = "YOUR_KEY_HERE"\nANTHROPIC_API_KEY = "sk-ant-xxxxxxxxxxxxxxxxxxxx"\n' > "$W/ph.txt" +out=$("$LL" "$W/ph.txt"); rc=$? +{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T7 placeholders not flagged" || bad T7 "$out" + +# T8: env-var REFERENCE (the correct pattern) NOT flagged +printf 'api_key = os.environ["OPENAI_API_KEY"]\nTOKEN="$GITHUB_TOKEN"\nkey: ${MY_SECRET}\n' > "$W/ref.txt" +out=$("$LL" "$W/ref.txt"); rc=$? +{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T8 env-var references not flagged" || bad T8 "$out" + +# T9: UUID / git-SHA assigned to a token name NOT flagged (generic-detector guard) +printf 'request_token = "550e8400-e29b-41d4-a716-446655440000"\ncommit_token = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0"\n' > "$W/uuid.txt" +out=$("$LL" "$W/uuid.txt"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T9 uuid + bare-hex not flagged as secrets" || bad T9 "$out" + +# T10: transcript mode flags a key in a tool_use INPUT (the key divergence from langcheck) +W="$W" GH="$GH" python3 -c ' +import json,os +key=os.environ["GH"] +rows=[ + {"type":"user","message":{"role":"user","content":[{"type":"text","text":"deploy it"}]}}, + {"type":"assistant","message":{"role":"assistant","content":[ + {"type":"text","text":"Running the deploy now."}, + {"type":"tool_use","name":"Bash","input":{"command":"curl -H \"Authorization: token "+key+"\" https://api.github.com"}} + ]}}, +] +open(os.environ["W"]+"/tx.jsonl","w").write("\n".join(json.dumps(r) for r in rows)) +' +out=$("$LL" --transcript "$W/tx.jsonl"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "tool_use:Bash" && ! echo "$out" | grep -qF "$GH"; } && ok "T10 transcript flags key in tool_use input, value redacted" || bad T10 "$out" + +# T11: stdin mode +out=$(printf 'nothing secret here\n' | "$LL"); echo "$out" | grep -q "clean" && ok "T11 stdin clean" || bad T11 "$out" +out=$(printf 'leaked %s now\n' "$AWS" | "$LL"); echo "$out" | grep -q "AWS access key" && ok "T11b stdin flags AWS key" || bad T11b "$out" + +# T12: --json structure +j=$(printf 'k=%s\n' "$GH" | "$LL" --json); echo "$j" | python3 -c 'import sys,json;d=json.load(sys.stdin);assert d["clean"] is False and d["findings"][0]["detector"]=="PROVIDER-KEY"' && ok "T12 --json well-formed" || bad T12 "$j" + +# T13: --help +"$LL" --help | grep -q "LEAKED SECRETS in LLM" && ok "T13 --help" || bad T13 "no help" + +# T14: missing file = exit 2 +"$LL" "$W/nope.txt" >/dev/null 2>&1; [ "$?" -eq 2 ] && ok "T14 missing file = exit 2" || bad T14 "rc" + +# T15: short value reveals zero prefix chars (redaction floor) +printf 'sk_live_%s\n' "$(python3 -c 'import random,string;print("".join(random.choice(string.ascii_letters+string.digits) for _ in range(24)))')" > "$W/short.txt" +out=$("$LL" "$W/short.txt"); echo "$out" | grep -q "sha256:" && ok "T15 fingerprint present" || bad T15 "$out" + +echo "-----"; echo "PASS=$PASS FAIL=$FAIL"; [ "$FAIL" -eq 0 ] diff --git a/spinlint/test_spinlint.sh b/spinlint/test_spinlint.sh new file mode 100755 index 0000000..e6db170 --- /dev/null +++ b/spinlint/test_spinlint.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Tests for spinlint. Ground truth = synthesized Claude-Code-shaped transcripts covering the +# real loop shapes (repeated identical call, repeated error, ping-pong edits) AND the +# productive-repetition negatives that must NOT trip it (retry that succeeds, batch reads, +# varying-input polling, monotonic edits). Built with python3 to match the real JSONL schema. +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" +SL="$HERE/spinlint"; chmod +x "$SL" +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +PASS=0; FAIL=0 +ok(){ PASS=$((PASS+1)); echo "PASS: $1"; } +bad(){ FAIL=$((FAIL+1)); echo "FAIL: $1 -- $2"; } + +mk(){ # mk + local out="$1"; shift + W="$W" OUT="$out" python3 -c " +import json,os +rows=[] +$* +open(os.environ['OUT'],'w').write('\n'.join(json.dumps(r) for r in rows)) +" +} +asst_tool(){ echo "rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':'$1','name':'$2','input':$3}]}})"; } +asst_text(){ echo "rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'text','text':'$1'}]}})"; } +result(){ echo "rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':'$1','is_error':$2,'content':$3}]}})"; } + +# T1: a clean, progressing session -> exit 0 +mk "$W/clean.jsonl" " +$(asst_tool t1 Read "{'file_path':'a.py'}") +$(result t1 False "'ok'") +$(asst_tool t2 Edit "{'file_path':'a.py','old_string':'x','new_string':'y'}") +$(result t2 False "'done'") +$(asst_tool t3 Bash "{'command':'pytest'}") +$(result t3 False "'2 passed'") +" +out=$("$SL" "$W/clean.jsonl"); rc=$? +{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T1 clean session = exit 0" || bad T1 "$out" + +# T2: 5x identical failing Bash -> REPEAT-CALL + exit 1 +mk "$W/repeat.jsonl" " +for i in range(5): + rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':f't{i}','name':'Bash','input':{'command':'npm run build'}}]}}) + rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':f't{i}','is_error':True,'content':\"error TS2304: name 'foo'\"}]}}) +" +out=$("$SL" "$W/repeat.jsonl"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "REPEAT-CALL" && echo "$out" | grep -q "all failing"; } && ok "T2 5x identical failing Bash flagged + exit 1" || bad T2 "$out" + +# T3: error->error->error (>=3 same error) -> REPEAT-ERROR +mk "$W/err.jsonl" " +for i in range(3): + rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':f'e{i}','name':'Edit','input':{'file_path':f'f{i}.py','old_string':'a','new_string':'b'}}]}}) + rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':f'e{i}','is_error':True,'content':'File has not been read yet. Read it first.'}]}}) +" +out=$("$SL" "$W/err.jsonl"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "REPEAT-ERROR"; } && ok "T3 repeated identical error flagged" || bad T3 "$out" + +# T4 (DEFINING): error -> error -> SUCCESS must NOT be flagged (success breaks the streak) +mk "$W/recover.jsonl" " +for i,err in enumerate([True,True,False]): + rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':f's{i}','name':'Bash','input':{'command':f'attempt {i}'}}]}}) + rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':f's{i}','is_error':err,'content':'compile error' if err else 'success'}]}}) +" +out=$("$SL" "$W/recover.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T4 DEFINING: error->error->success NOT flagged" || bad T4 "$out" + +# T5: batch reads of DIFFERENT files -> NOT flagged +mk "$W/batch.jsonl" " +for i in range(8): + rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':f'b{i}','name':'Read','input':{'file_path':f'src/mod{i}.py'}}]}}) + rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':f'b{i}','content':'contents'}]}}) +" +out=$("$SL" "$W/batch.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T5 batch reads of different files NOT flagged" || bad T5 "$out" + +# T6: polling with VARYING input -> NOT flagged +mk "$W/poll.jsonl" " +for i in range(6): + rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':f'p{i}','name':'Bash','input':{'command':f'sleep {i} && curl host?cursor={i}'}}]}}) + rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':f'p{i}','content':'pending'}]}}) +" +out=$("$SL" "$W/poll.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T6 varying-input polling NOT flagged" || bad T6 "$out" + +# T7: PING-PONG oscillating edits to one file -> flagged +mk "$W/pingpong.jsonl" " +edits=[('A','B'),('B','A'),('A','B'),('B','A')] +for i,(o,n) in enumerate(edits): + rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':f'pp{i}','name':'Edit','input':{'file_path':'src/auth.go','old_string':o,'new_string':n}}]}}) + rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':f'pp{i}','content':'edited'}]}}) +" +out=$("$SL" "$W/pingpong.jsonl"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "PING-PONG" && echo "$out" | grep -q "auth.go"; } && ok "T7 oscillating edits flagged" || bad T7 "$out" + +# T8: monotonic forward edits to one file (all distinct) -> NOT flagged +mk "$W/forward.jsonl" " +for i in range(6): + rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':f'fw{i}','name':'Edit','input':{'file_path':'big.py','old_string':f'line{i}', 'new_string':f'newline{i}'}}]}}) + rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':f'fw{i}','content':'edited'}]}}) +" +out=$("$SL" "$W/forward.jsonl"); rc=$? +{ [ "$rc" -eq 0 ]; } && ok "T8 monotonic forward edits NOT flagged" || bad T8 "$out" + +# T9: --json well-formed + carries no raw command beyond truncated signature +mk "$W/j.jsonl" " +for i in range(4): + rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':f'j{i}','name':'Bash','input':{'command':'go test ./...'}}]}}) + rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':f'j{i}','is_error':True,'content':'FAIL'}]}}) +" +j=$("$SL" --json "$W/j.jsonl") +echo "$j" | python3 -c 'import sys,json;d=json.load(sys.stdin);assert d["clean"] is False and d["estimated_wasted_calls"]>=3 and d["findings"][0]["detector"]=="REPEAT-CALL"' && ok "T9 --json well-formed + wasted count" || bad T9 "$j" + +# T10: stdin mode +out=$(cat "$W/repeat.jsonl" | "$SL"); echo "$out" | grep -q "REPEAT-CALL" && ok "T10 stdin mode" || bad T10 "$out" + +# T11: --repeat threshold tunable (4 identical, default 3 flags; --repeat 5 does NOT) +mk "$W/four.jsonl" " +for i in range(4): + rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':f'q{i}','name':'Bash','input':{'command':'ls'}}]}}) + rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':f'q{i}','content':'ok'}]}}) +" +"$SL" "$W/four.jsonl" >/dev/null 2>&1; d3=$? +"$SL" --repeat 5 "$W/four.jsonl" >/dev/null 2>&1; d5=$? +{ [ "$d3" -ne 0 ] && [ "$d5" -eq 0 ]; } && ok "T11 --repeat threshold tunable" || bad T11 "default=$d3 repeat5=$d5" + +# T12: --help +"$SL" --help | grep -q "WASTED-LOOP" && ok "T12 --help" || bad T12 "no help" + +# T13: missing file = exit 2 +"$SL" "$W/nope.jsonl" >/dev/null 2>&1; [ "$?" -eq 2 ] && ok "T13 missing file = exit 2" || bad T13 "rc" + +# T14: list-shaped tool_result.content parsed (error in list content still counts) +mk "$W/listcontent.jsonl" " +for i in range(3): + rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':f'l{i}','name':'Bash','input':{'command':'make'}}]}}) + rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':f'l{i}','is_error':True,'content':[{'type':'text','text':'make: *** error'}]}]}}) +" +out=$("$SL" "$W/listcontent.jsonl"); rc=$? +{ [ "$rc" -ne 0 ] && echo "$out" | grep -qE "REPEAT-(CALL|ERROR)"; } && ok "T14 list-shaped tool_result content parsed" || bad T14 "$out" + +# T15: wasted-calls estimate present in human output +out=$("$SL" "$W/repeat.jsonl"); echo "$out" | grep -q "wasted tool-calls" && ok "T15 wasted-calls estimate shown" || bad T15 "$out" + +# T16: not-a-transcript = exit 2 +printf 'just some text\nnot jsonl\n' > "$W/plain.txt" +"$SL" "$W/plain.txt" >/dev/null 2>&1; [ "$?" -eq 2 ] && ok "T16 non-transcript = exit 2" || bad T16 "rc" + +echo "-----"; echo "PASS=$PASS FAIL=$FAIL"; [ "$FAIL" -eq 0 ] diff --git a/storyboard-doctor/README.md b/storyboard-doctor/README.md index 2ad5c6a..9e18982 100644 --- a/storyboard-doctor/README.md +++ b/storyboard-doctor/README.md @@ -20,4 +20,6 @@ Exit 0 = clean, 1 = problems. Built from a session where these exact issues forc Tests: `./test_storyboard_doctor.sh` (10/10). +Built and run by [deemwar](https://deemwar.com). Putting agents to work on code that matters? **[Talk to us](https://deemwar.com/contact).** + MIT © deemwar