deemwar-products · muthuishere · Jun 25, 2026
diff --git a/claimlint/test_claimlint.sh b/claimlint/test_claimlint.sh
@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+# Tests for claimlint. Ground truth = synthesized Claude-Code-shaped transcripts pairing the
+# SAME completion claim with a session that does vs. doesn't support it — plus the FP guards
+# (future tense, conditional) that must NOT trip, and --root filesystem verification.
+set -u
+HERE="$(cd "$(dirname "$0")" && pwd)"
+CL="$HERE/claimlint"; chmod +x "$CL"
+W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT
+PASS=0; FAIL=0
+ok(){ PASS=$((PASS+1)); echo "PASS: $1"; }
+bad(){ FAIL=$((FAIL+1)); echo "FAIL: $1 -- $2"; }
+
+mk(){ local out="$1"; shift; W="$W" OUT="$out" python3 -c "
+import json,os
+rows=[]
+$*
+open(os.environ['OUT'],'w').write('\n'.join(json.dumps(r) for r in rows))
+"; }
+# helpers usable inside mk bodies
+A_TEXT(){ echo "rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'text','text':\"$1\"}]}})"; }
+A_TOOL(){ echo "rows.append({'type':'assistant','message':{'role':'assistant','content':[{'type':'tool_use','id':'$1','name':'$2','input':$3}]}})"; }
+RESULT(){ echo "rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':'$1','is_error':$2,'content':$3}]}})"; }
+
+# T1: HONEST — claim backed by a real passing test run + a real Write -> exit 0
+mk "$W/honest.jsonl" "
+$(A_TOOL w1 Write "{'file_path':'src/auth.py','content':'def login(): ...'}")
+$(RESULT w1 False "'File created successfully'")
+$(A_TOOL b1 Bash "{'command':'pytest -q'}")
+$(RESULT b1 False "'3 passed in 0.4s'")
+$(A_TEXT 'All tests pass and I created src/auth.py. Done.')
+"
+out=$("$CL" "$W/honest.jsonl"); rc=$?
+{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T1 honest (test+file backed) = exit 0" || bad T1 "$out"
+
+# T2: LYING — claims pass + created, neither happened -> both findings, exit 1
+mk "$W/lying.jsonl" "
+$(A_TOOL r1 Read "{'file_path':'main.py'}")
+$(RESULT r1 False "'code'")
+$(A_TEXT 'All tests pass and I created src/auth.py. Done.')
+"
+out=$("$CL" "$W/lying.jsonl"); rc=$?
+{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "TEST-CLAIM" && echo "$out" | grep -q "FILE-CLAIM"; } && ok "T2 lying (no test, no write) flags both + exit 1" || bad T2 "$out"
+
+# T3: FP — FUTURE tense ('I will create / run the tests') must NOT flag
+mk "$W/future.jsonl" "
+$(A_TOOL r1 Read "{'file_path':'main.py'}")
+$(RESULT r1 False "'code'")
+$(A_TEXT 'Next I will create src/auth.py and then the tests should pass.')
+"
+out=$("$CL" "$W/future.jsonl"); rc=$?
+{ [ "$rc" -eq 0 ]; } && ok "T3 future-tense plan NOT flagged" || bad T3 "$out"
+
+# T4: FP — CONDITIONAL ('if the tests pass') must NOT flag
+mk "$W/cond.jsonl" "
+$(A_TOOL r1 Read "{'file_path':'main.py'}")
+$(RESULT r1 False "'code'")
+$(A_TEXT 'If the tests pass we are done; run them to confirm.')
+"
+out=$("$CL" "$W/cond.jsonl"); rc=$?
+{ [ "$rc" -eq 0 ]; } && ok "T4 conditional claim NOT flagged" || bad T4 "$out"
+
+# T5: TEST-CLAIM — claims pass but the test run ERRORED -> flagged
+mk "$W/testfail.jsonl" "
+$(A_TOOL b1 Bash "{'command':'pytest -q'}")
+$(RESULT b1 True "'2 failed, 1 passed'")
+$(A_TEXT 'All tests pass now.')
+"
+out=$("$CL" "$W/testfail.jsonl"); rc=$?
+{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "TEST-CLAIM"; } && ok "T5 claim-pass but test failed flagged" || bad T5 "$out"
+
+# T6: FILE-CLAIM basename match — claim 'auth.py', Write was 'src/auth.py' -> NOT flagged
+mk "$W/basename.jsonl" "
+$(A_TOOL w1 Write "{'file_path':'src/auth.py','content':'x'}")
+$(RESULT w1 False "'ok'")
+$(A_TEXT 'I created auth.py for you.')
+"
+out=$("$CL" "$W/basename.jsonl"); rc=$?
+{ [ "$rc" -eq 0 ]; } && ok "T6 file claim matched by basename NOT flagged" || bad T6 "$out"
+
+# T7: --root — claimed file exists ON DISK though no edit tool call -> NOT flagged
+mkdir -p "$W/repo/lib"; echo "x" > "$W/repo/lib/util.py"
+mk "$W/disk.jsonl" "
+$(A_TOOL r1 Read "{'file_path':'main.py'}")
+$(RESULT r1 False "'code'")
+$(A_TEXT 'I updated lib/util.py.')
+"
+out=$("$CL" --root "$W/repo" "$W/disk.jsonl"); rc=$?
+{ [ "$rc" -eq 0 ]; } && ok "T7 --root: file on disk NOT flagged" || bad T7 "$out"
+
+# T8: --root — claimed file NOT edited and NOT on disk -> flagged
+mk "$W/ghost.jsonl" "
+$(A_TOOL r1 Read "{'file_path':'main.py'}")
+$(RESULT r1 False "'code'")
+$(A_TEXT 'I created lib/ghost.py with the helper.')
+"
+out=$("$CL" --root "$W/repo" "$W/ghost.jsonl"); rc=$?
+{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "FILE-CLAIM"; } && ok "T8 --root: ghost file flagged" || bad T8 "$out"
+
+# T9: non-file tokens (e.g., version numbers) NOT treated as file claims
+mk "$W/notfile.jsonl" "
+$(A_TOOL b1 Bash "{'command':'pytest -q'}")
+$(RESULT b1 False "'5 passed'")
+$(A_TEXT 'All tests pass. Updated to v2.1 and documented e.g. the flow.')
+"
+out=$("$CL" "$W/notfile.jsonl"); rc=$?
+{ [ "$rc" -eq 0 ]; } && ok "T9 version/e.g. not treated as files" || bad T9 "$out"
+
+# T10: stdin mode
+out=$(cat "$W/lying.jsonl" | "$CL"); echo "$out" | grep -q "FILE-CLAIM" && ok "T10 stdin mode" || bad T10 "$out"
+
+# T11: --json well-formed
+j=$("$CL" --json "$W/lying.jsonl"); echo "$j" | python3 -c 'import sys,json;d=json.load(sys.stdin);assert d["clean"] is False and len(d["findings"])>=2' && ok "T11 --json well-formed" || bad T11 "$j"
+
+# T12: --help
+"$CL" --help | grep -q "completion CLAIMS" && ok "T12 --help" || bad T12 "no help"
+
+# T13: missing file = exit 2
+"$CL" "$W/nope.jsonl" >/dev/null 2>&1; [ "$?" -eq 2 ] && ok "T13 missing file = exit 2" || bad T13 "rc"
+
+# T14: not-a-transcript = exit 2
+printf 'hello world\n' > "$W/plain.txt"; "$CL" "$W/plain.txt" >/dev/null 2>&1; [ "$?" -eq 2 ] && ok "T14 non-transcript = exit 2" || bad T14 "rc"
+
+# T15: honest 'Done.' with no specific claim and supporting edit -> exit 0 (no false alarm on bare 'Done')
+mk "$W/baredone.jsonl" "
+$(A_TOOL w1 Edit "{'file_path':'README.md','old_string':'a','new_string':'b'}")
+$(RESULT w1 False "'ok'")
+$(A_TEXT 'Done.')
+"
+out=$("$CL" "$W/baredone.jsonl"); rc=$?
+{ [ "$rc" -eq 0 ]; } && ok "T15 bare 'Done.' not falsely flagged" || bad T15 "$out"
+
+# T16: list-shaped tool_result content parsed (passing test in list content)
+mk "$W/listtest.jsonl" "
+$(A_TOOL b1 Bash "{'command':'go test ./...'}")
+rows.append({'type':'user','message':{'role':'user','content':[{'type':'tool_result','tool_use_id':'b1','is_error':False,'content':[{'type':'text','text':'ok  all 4 passed'}]}]}})
+$(A_TEXT 'All tests pass.')
+"
+out=$("$CL" "$W/listtest.jsonl"); rc=$?
+{ [ "$rc" -eq 0 ]; } && ok "T16 list-content passing test supports claim" || bad T16 "$out"
+
+echo "-----"; echo "PASS=$PASS FAIL=$FAIL"; [ "$FAIL" -eq 0 ]
diff --git a/langcheck/test_langcheck.sh b/langcheck/test_langcheck.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Tests for langcheck. Ground truth = real text samples (the actual bug shapes:
+# CJK in English output, mojibake, runaway repetition) + a real transcript JSONL.
+set -u
+HERE="$(cd "$(dirname "$0")" && pwd)"
+LC="$HERE/langcheck"; chmod +x "$LC"
+W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT
+PASS=0; FAIL=0
+ok(){ PASS=$((PASS+1)); echo "PASS: $1"; }
+bad(){ FAIL=$((FAIL+1)); echo "FAIL: $1 -- $2"; }
+
+# T1: clean English -> exit 0, "clean"
+printf 'The quick brown fox jumps over the lazy dog.\nThis is a normal paragraph of English output.\n' > "$W/clean.txt"
+out=$("$LC" "$W/clean.txt"); rc=$?
+{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T1 clean English = exit 0" || bad T1 "$out"
+
+# T2: English output with a random CJK run (the actual bug) -> SCRIPT-INTRUSION, exit 1
+python3 -c 'open("'"$W/cjk.txt"'","w",encoding="utf-8").write("Here is the function you asked for. 这是一个 it returns the value.\nAll good otherwise.\n")'
+out=$("$LC" "$W/cjk.txt"); rc=$?
+{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "SCRIPT-INTRUSION" && echo "$out" | grep -q "CJK"; } && ok "T2 CJK-in-English flagged + exit 1" || bad T2 "$out"
+
+# T3: a genuinely Chinese document is NOT flagged as intrusion
+python3 -c 'open("'"$W/zh.txt"'","w",encoding="utf-8").write("这是一个完全用中文写的文档。它包含许多中文句子和段落。没有英文。\n第二行也是中文内容。\n")'
+out=$("$LC" "$W/zh.txt"); rc=$?
+{ [ "$rc" -eq 0 ] && ! echo "$out" | grep -q "SCRIPT-INTRUSION"; } && ok "T3 predominantly-Chinese doc not false-flagged" || bad T3 "$out"
+
+# T4: mojibake replacement char -> MOJIBAKE
+python3 -c 'open("'"$W/moji.txt"'","w",encoding="utf-8").write("The price is �25 today and the rest is fine.\n")'
+out=$("$LC" "$W/moji.txt")
+echo "$out" | grep -q "MOJIBAKE" && ok "T4 mojibake (U+FFFD) flagged" || bad T4 "$out"
+
+# T5: runaway repetition -> REPETITION
+{ echo "Generating the report now."; for i in 1 2 3 4 5; do echo "I will now continue."; done; echo "Done."; } > "$W/rep.txt"
+out=$("$LC" "$W/rep.txt")
+echo "$out" | grep -q "REPETITION" && echo "$out" | grep -q "5x" && ok "T5 runaway repetition flagged (5x)" || bad T5 "$out"
+
+# T5b: below threshold (3x default 4) NOT flagged
+{ for i in 1 2 3; do echo "ok line"; done; echo "end"; } > "$W/rep2.txt"
+out=$("$LC" "$W/rep2.txt"); echo "$out" | grep -q "REPETITION" && bad T5b "3x wrongly flagged" || ok "T5b under-threshold repetition not flagged"
+
+# T6: emoji / accented Latin in English NOT flagged as foreign-script
+python3 -c 'open("'"$W/emoji.txt"'","w",encoding="utf-8").write("Great work \U0001F389 the café is naïve about résumés.\n")'
+out=$("$LC" "$W/emoji.txt"); rc=$?
+{ [ "$rc" -eq 0 ] && ! echo "$out" | grep -q "SCRIPT-INTRUSION"; } && ok "T6 emoji + accented Latin not flagged" || bad T6 "$out"
+
+# T7: transcript mode extracts assistant text + flags the CJK in it
+python3 -c '
+import json
+rows=[
+ {"type":"user","message":{"role":"user","content":[{"type":"text","text":"hi"}]}},
+ {"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Sure, here is the plan 部署 and the steps."}]}},
+]
+open("'"$W/tx.jsonl"'","w").write("\n".join(json.dumps(r) for r in rows))
+'
+out=$("$LC" --transcript "$W/tx.jsonl"); rc=$?
+{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "SCRIPT-INTRUSION"; } && ok "T7 transcript mode flags CJK in assistant text" || bad T7 "$out"
+
+# T8: stdin mode
+out=$(printf 'normal english here\n' | "$LC"); echo "$out" | grep -q "clean" && ok "T8 stdin clean" || bad T8 "$out"
+out=$(python3 -c 'print("english then 한국어 hangul intrusion")' | "$LC"); echo "$out" | grep -q "Hangul" && ok "T8b stdin flags Hangul intrusion" || bad T8b "$out"
+
+# T9: --help
+"$LC" --help | grep -q "contamination in LLM" && ok "T9 --help" || bad T9 "no help"
+
+# T10: missing file = exit 2
+"$LC" "$W/nope.txt" >/dev/null 2>&1; [ "$?" -eq 2 ] && ok "T10 missing file = exit 2" || bad T10 "rc"
+
+echo "-----"; echo "PASS=$PASS FAIL=$FAIL"; [ "$FAIL" -eq 0 ]
diff --git a/leaklint/test_leaklint.sh b/leaklint/test_leaklint.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# Tests for leaklint. Ground truth = real secret SHAPES (vendor key formats, PEM headers,
+# KEY=<random> assignments) + placeholder/env-ref negatives + a transcript with a key in a
+# tool_use input. The DEFINING test asserts the tool never echoes a secret's value.
+# NOTE: all "secrets" below are obviously-fake random strings generated at test time — no
+# real credential is ever hardcoded here.
+set -u
+HERE="$(cd "$(dirname "$0")" && pwd)"
+LL="$HERE/leaklint"; chmod +x "$LL"
+W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT
+PASS=0; FAIL=0
+ok(){ PASS=$((PASS+1)); echo "PASS: $1"; }
+bad(){ FAIL=$((FAIL+1)); echo "FAIL: $1 -- $2"; }
+
+# fake-but-shaped secrets (random tails so they trip format/entropy but are not real)
+ANT="sk-ant-api03-$(python3 -c 'import random,string;print("".join(random.choice(string.ascii_letters+string.digits+"_-") for _ in range(90)))')"
+GH="ghp_$(python3 -c 'import random,string;print("".join(random.choice(string.ascii_letters+string.digits) for _ in range(36)))')"
+AWS="AKIA$(python3 -c 'import random,string;print("".join(random.choice(string.ascii_uppercase+string.digits) for _ in range(16)))')"
+RAND="$(python3 -c 'import random,string;print("".join(random.choice(string.ascii_letters+string.digits) for _ in range(40)))')"
+
+# T1: clean prose -> exit 0, "clean"
+printf 'Here is the function you asked for. It reads the key from the environment.\nAll good.\n' > "$W/clean.txt"
+out=$("$LL" "$W/clean.txt"); rc=$?
+{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T1 clean prose = exit 0" || bad T1 "$out"
+
+# T2: Anthropic key -> PROVIDER-KEY, exit 1
+printf 'Sure, use this key: %s in your config.\n' "$ANT" > "$W/ant.txt"
+out=$("$LL" "$W/ant.txt"); rc=$?
+{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "PROVIDER-KEY" && echo "$out" | grep -q "Anthropic"; } && ok "T2 Anthropic key flagged + exit 1" || bad T2 "$out"
+
+# T3 (DEFINING): the secret VALUE never appears in stdout, in any mode
+out=$("$LL" "$W/ant.txt"); j=$("$LL" --json "$W/ant.txt")
+if echo "$out$j" | grep -qF "$ANT"; then bad T3 "secret value LEAKED into output"; else ok "T3 DEFINING: value never printed (text+json)"; fi
+# and the json must carry no "value" key
+echo "$j" | grep -q '"value"' && bad T3b "json has a value key" || ok "T3b json has no value key"
+
+# T4: GitHub token + AWS key both flagged
+printf 'token=%s and aws=%s\n' "$GH" "$AWS" > "$W/multi.txt"
+out=$("$LL" "$W/multi.txt"); rc=$?
+{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "GitHub token" && echo "$out" | grep -q "AWS access key"; } && ok "T4 GitHub + AWS flagged" || bad T4 "$out"
+
+# T5: PEM private-key header flagged, body bytes never printed
+printf -- '-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAAB\n-----END OPENSSH PRIVATE KEY-----\n' > "$W/pem.txt"
+out=$("$LL" "$W/pem.txt")
+{ echo "$out" | grep -q "PRIVATE-KEY" && ! echo "$out" | grep -q "b3BlbnNz"; } && ok "T5 PEM header flagged, body not printed" || bad T5 "$out"
+
+# T6: high-entropy KEY=<literal> assignment flagged (ASSIGNED-SECRET)
+printf 'OPENAI_API_KEY = "%s"\n' "$RAND" > "$W/assign.txt"
+out=$("$LL" "$W/assign.txt")
+echo "$out" | grep -q "ASSIGNED-SECRET" && ok "T6 KEY=<random literal> flagged" || bad T6 "$out"
+
+# T7: placeholder NOT flagged
+printf 'OPENAI_API_KEY = "YOUR_KEY_HERE"\nANTHROPIC_API_KEY = "sk-ant-xxxxxxxxxxxxxxxxxxxx"\n' > "$W/ph.txt"
+out=$("$LL" "$W/ph.txt"); rc=$?
+{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T7 placeholders not flagged" || bad T7 "$out"
+
+# T8: env-var REFERENCE (the correct pattern) NOT flagged
+printf 'api_key = os.environ["OPENAI_API_KEY"]\nTOKEN="$GITHUB_TOKEN"\nkey: ${MY_SECRET}\n' > "$W/ref.txt"
+out=$("$LL" "$W/ref.txt"); rc=$?
+{ [ "$rc" -eq 0 ] && echo "$out" | grep -q "clean"; } && ok "T8 env-var references not flagged" || bad T8 "$out"
+
+# T9: UUID / git-SHA assigned to a token name NOT flagged (generic-detector guard)
+printf 'request_token = "550e8400-e29b-41d4-a716-446655440000"\ncommit_token = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0"\n' > "$W/uuid.txt"
+out=$("$LL" "$W/uuid.txt"); rc=$?
+{ [ "$rc" -eq 0 ]; } && ok "T9 uuid + bare-hex not flagged as secrets" || bad T9 "$out"
+
+# T10: transcript mode flags a key in a tool_use INPUT (the key divergence from langcheck)
+W="$W" GH="$GH" python3 -c '
+import json,os
+key=os.environ["GH"]
+rows=[
+ {"type":"user","message":{"role":"user","content":[{"type":"text","text":"deploy it"}]}},
+ {"type":"assistant","message":{"role":"assistant","content":[
+   {"type":"text","text":"Running the deploy now."},
+   {"type":"tool_use","name":"Bash","input":{"command":"curl -H \"Authorization: token "+key+"\" https://api.github.com"}}
+ ]}},
+]
+open(os.environ["W"]+"/tx.jsonl","w").write("\n".join(json.dumps(r) for r in rows))
+'
+out=$("$LL" --transcript "$W/tx.jsonl"); rc=$?
+{ [ "$rc" -ne 0 ] && echo "$out" | grep -q "tool_use:Bash" && ! echo "$out" | grep -qF "$GH"; } && ok "T10 transcript flags key in tool_use input, value redacted" || bad T10 "$out"
+
+# T11: stdin mode
+out=$(printf 'nothing secret here\n' | "$LL"); echo "$out" | grep -q "clean" && ok "T11 stdin clean" || bad T11 "$out"
+out=$(printf 'leaked %s now\n' "$AWS" | "$LL"); echo "$out" | grep -q "AWS access key" && ok "T11b stdin flags AWS key" || bad T11b "$out"
+
+# T12: --json structure
+j=$(printf 'k=%s\n' "$GH" | "$LL" --json); echo "$j" | python3 -c 'import sys,json;d=json.load(sys.stdin);assert d["clean"] is False and d["findings"][0]["detector"]=="PROVIDER-KEY"' && ok "T12 --json well-formed" || bad T12 "$j"
+
+# T13: --help
+"$LL" --help | grep -q "LEAKED SECRETS in LLM" && ok "T13 --help" || bad T13 "no help"
+
+# T14: missing file = exit 2
+"$LL" "$W/nope.txt" >/dev/null 2>&1; [ "$?" -eq 2 ] && ok "T14 missing file = exit 2" || bad T14 "rc"
+
+# T15: short value reveals zero prefix chars (redaction floor)
+printf 'sk_live_%s\n' "$(python3 -c 'import random,string;print("".join(random.choice(string.ascii_letters+string.digits) for _ in range(24)))')" > "$W/short.txt"
+out=$("$LL" "$W/short.txt"); echo "$out" | grep -q "sha256:" && ok "T15 fingerprint present" || bad T15 "$out"
+
+echo "-----"; echo "PASS=$PASS FAIL=$FAIL"; [ "$FAIL" -eq 0 ]