From 417d852dd4de1b131ffd8093cb482d1db6a6b9b9 Mon Sep 17 00:00:00 2001 From: itswl Date: Mon, 2 Nov 2026 16:43:04 +0800 Subject: [PATCH 1/2] =?UTF-8?q?deps:=20the=20investigator's=20runtime=20is?= =?UTF-8?q?=20relocked=20for=20the=20first=20time=20since=20August=20?= =?UTF-8?q?=E2=80=94=20claude-agent-sdk=200.2.176,=20fastapi=200.162.0,=20?= =?UTF-8?q?uvicorn=200.54.0;=20pytest=209.2.0,=20ruff=200.17.3,=20mypy=202?= =?UTF-8?q?.4.1=20everywhere=20=E2=80=94=20Dependabot=20#36=20#37=20#38?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dependabot's first proposal for the investigator's runtime since its strategy was fixed (#35), with the pipe's and the judge's test tooling, relocked as one change. Exact pins follow the bot and floors take the newest, so the SDK is 0.2.176 where the bot proposed 0.2.175; the newer release only updates the bundled CLI. Each version has its upstream tag, and nothing else moved in the locks. The setuptools build floor rises to 84.2.1 as proposed. The SDK wheel bundles the CLI the engine runs and looks for it first, so this also moves the investigator's agent runtime from CLI 2.1.235 to 2.1.316; the npm CLI the Dockerfile pins stays on PATH unused, as it was in 0.9.2. The changelog for 0.2.141 to 0.2.176 is bundled-CLI updates plus additive options and typing, and mypy checks the engine's options against 0.2.176. --- hookjudge/requirements.lock | 6 +++--- hookjudge/requirements.txt | 6 +++--- hookprobe/pyproject.toml | 8 ++++---- hookprobe/requirements.lock | 12 ++++++------ hookprobe/requirements.txt | 12 ++++++------ hookrelay/requirements.lock | 6 +++--- hookrelay/requirements.txt | 6 +++--- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/hookjudge/requirements.lock b/hookjudge/requirements.lock index 19e2037b..1deaab44 100644 --- a/hookjudge/requirements.lock +++ b/hookjudge/requirements.lock @@ -25,7 +25,7 @@ license-expression==30.4.4 markdown-it-py==4.2.0 mdurl==0.1.2 msgpack==1.2.4 -mypy==2.4.0 +mypy==2.4.1 mypy_extensions==1.1.0 opentelemetry-api==1.45.1 packageurl-python==0.17.6 @@ -41,14 +41,14 @@ pydantic==2.13.4 pydantic_core==2.46.4 Pygments==2.21.0 pyparsing==3.3.2 -pytest==9.1.1 +pytest==9.2.0 pytest-asyncio==1.4.1.post1 pytest-cov==7.1.0 python-dotenv==1.2.3 PyYAML==6.0.3 requests==2.34.2 rich==15.0.0 -ruff==0.17.2 +ruff==0.17.3 sortedcontainers==2.4.0 starlette==1.6.0 stevedore==5.9.0 diff --git a/hookjudge/requirements.txt b/hookjudge/requirements.txt index fef34d19..991787eb 100644 --- a/hookjudge/requirements.txt +++ b/hookjudge/requirements.txt @@ -5,10 +5,10 @@ aiosqlite>=0.22.1 httpx>=0.28.1 # Test + lint -pytest>=9.1.1 +pytest>=9.2.0 pytest-asyncio>=1.4.1 pytest-cov>=7.1.0 -ruff==0.17.2 +ruff==0.17.3 bandit>=1.9.4 pip-audit>=2.10.1 -mypy>=2.4.0 +mypy>=2.4.1 diff --git a/hookprobe/pyproject.toml b/hookprobe/pyproject.toml index 900b9cdb..d30ddeb0 100644 --- a/hookprobe/pyproject.toml +++ b/hookprobe/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68"] +requires = ["setuptools>=84.2.1"] build-backend = "setuptools.build_meta" [project] @@ -9,9 +9,9 @@ description = "Single-purpose deep-analysis agent runner: OpenClaw-compatible co readme = "README.md" requires-python = ">=3.12" dependencies = [ - "fastapi>=0.115", - "uvicorn[standard]>=0.34", - "claude-agent-sdk>=0.1.0", + "fastapi>=0.162.0", + "uvicorn[standard]>=0.54.0", + "claude-agent-sdk>=0.2.175", ] [tool.setuptools] diff --git a/hookprobe/requirements.lock b/hookprobe/requirements.lock index 3840b440..d206bc3e 100644 --- a/hookprobe/requirements.lock +++ b/hookprobe/requirements.lock @@ -9,13 +9,13 @@ CacheControl==0.14.4 certifi==2026.10.19 cffi==2.1.1 charset-normalizer==3.5.1 -claude-agent-sdk==0.2.140 +claude-agent-sdk==0.2.176 click==8.4.2 coverage==7.16.2 cryptography==50.0.0 cyclonedx-python-lib==11.12.0 defusedxml==0.7.1 -fastapi==0.141.1 +fastapi==0.162.0 filelock==4.2.4 h11==0.16.0 httpcore==1.0.9 @@ -34,7 +34,7 @@ mcp==2.3.0 mcp-types==2.3.0 mdurl==0.1.2 msgpack==1.2.4 -mypy==2.4.0 +mypy==2.4.1 mypy_extensions==1.1.0 opentelemetry-api==1.45.1 packageurl-python==0.17.6 @@ -52,7 +52,7 @@ pydantic_core==2.46.4 Pygments==2.21.0 PyJWT==2.15.1 pyparsing==3.3.2 -pytest==9.1.1 +pytest==9.2.0 pytest-cov==7.1.0 python-dotenv==1.2.3 python-multipart==0.0.32 @@ -61,7 +61,7 @@ referencing==0.37.0 requests==2.34.2 rich==15.0.0 rpds-py==2026.6.3 -ruff==0.17.2 +ruff==0.17.3 sniffio==1.3.1 sortedcontainers==2.4.0 sse-starlette==3.4.8 @@ -73,7 +73,7 @@ truststore==0.10.4 typing-inspection==0.4.4 typing_extensions==4.16.0 urllib3==2.8.0 -uvicorn==0.52.4 +uvicorn==0.54.0 uvloop==0.22.1 watchfiles==1.2.0 websockets==17.0.1 diff --git a/hookprobe/requirements.txt b/hookprobe/requirements.txt index 73cf0e42..4eb6f399 100644 --- a/hookprobe/requirements.txt +++ b/hookprobe/requirements.txt @@ -1,15 +1,15 @@ # Three runtime dependencies, chosen to stay three. The agent loop, built-in # tools, MCP client and SKILL.md loading all come from the Claude Agent SDK — # hookprobe itself is only a contract shell around one engine call. -fastapi>=0.115 -uvicorn[standard]>=0.34 -claude-agent-sdk>=0.1.0 +fastapi>=0.162.0 +uvicorn[standard]>=0.54.0 +claude-agent-sdk>=0.2.175 # Test + lint (tests never import the SDK; they inject fake engines) -pytest>=9.1.1 +pytest>=9.2.0 httpx>=0.28.1 pytest-cov>=7.1.0 -ruff==0.17.2 +ruff==0.17.3 bandit>=1.9.4 pip-audit>=2.10.1 -mypy>=2.4.0 +mypy>=2.4.1 diff --git a/hookrelay/requirements.lock b/hookrelay/requirements.lock index 96f9ca56..0b056887 100644 --- a/hookrelay/requirements.lock +++ b/hookrelay/requirements.lock @@ -25,7 +25,7 @@ license-expression==30.4.4 markdown-it-py==4.2.0 mdurl==0.1.2 msgpack==1.2.4 -mypy==2.4.0 +mypy==2.4.1 mypy_extensions==1.1.0 opentelemetry-api==1.45.1 packageurl-python==0.17.6 @@ -41,14 +41,14 @@ pydantic==2.13.4 pydantic_core==2.46.4 Pygments==2.21.0 pyparsing==3.3.2 -pytest==9.1.1 +pytest==9.2.0 pytest-asyncio==1.4.1.post1 pytest-cov==7.1.0 python-dotenv==1.2.3 PyYAML==6.0.3 requests==2.34.2 rich==15.0.0 -ruff==0.17.2 +ruff==0.17.3 sortedcontainers==2.4.0 starlette==1.6.0 stevedore==5.9.0 diff --git a/hookrelay/requirements.txt b/hookrelay/requirements.txt index 97c3d441..658d97cb 100644 --- a/hookrelay/requirements.txt +++ b/hookrelay/requirements.txt @@ -6,11 +6,11 @@ httpx>=0.28.1 pyyaml>=6.0.3 # Test + lint -pytest>=9.1.1 +pytest>=9.2.0 pytest-asyncio>=1.4.1 pytest-cov>=7.1.0 -ruff==0.17.2 +ruff==0.17.3 bandit>=1.9.4 pip-audit>=2.10.1 -mypy>=2.4.0 +mypy>=2.4.1 types-PyYAML From 44f75b8d5ce2e42076f0205b0bd623afe8e02cdc Mon Sep 17 00:00:00 2001 From: itswl Date: Mon, 2 Nov 2026 16:43:04 +0800 Subject: [PATCH 2/2] =?UTF-8?q?hookprobe:=20the=20message=20reaches=20the?= =?UTF-8?q?=20model=20as=20words=20=E2=80=94=20an=20@path=20in=20alert=20o?= =?UTF-8?q?r=20chat=20text=20no=20longer=20makes=20the=20CLI=20read=20a=20?= =?UTF-8?q?file=20the=20guard=20never=20sees?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With verbatim_prompts the CLI takes the message as text, not as input typed at its prompt. Before it, an `@path` anywhere in an alert or a chat reply made the CLI read that file and hand it to the model as a Read result no PreToolUse hook saw, and a message starting with `/` ran a CLI command instead of an investigation. Measured on the 0.9.2 image and on this change's image with a recording stand-in for the model API. After the switch neither happens, and the scripted turn still runs with the guard refusing a protected write. The test pins the option, the guard mutation list flips it, and the containment table's guard row says what it stops. The decision note records the measurement and one finding left for the operator: the engine runs the SDK's bundled CLI, not the npm one the Dockerfile pins. --- ...-the-message-reaches-the-model-as-words.md | 62 +++++++++++++++++++ docs/containment.md | 2 +- hookprobe/hookprobe/engine.py | 12 ++++ hookprobe/tests/test_engine_loop.py | 16 +++++ scripts/assert_guards_are_tested.py | 9 ++- 5 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 .agents/notes/implemented/2026-11-02-the-message-reaches-the-model-as-words.md diff --git a/.agents/notes/implemented/2026-11-02-the-message-reaches-the-model-as-words.md b/.agents/notes/implemented/2026-11-02-the-message-reaches-the-model-as-words.md new file mode 100644 index 00000000..4c7bb1ee --- /dev/null +++ b/.agents/notes/implemented/2026-11-02-the-message-reaches-the-model-as-words.md @@ -0,0 +1,62 @@ +--- +title: The message reaches the model as words, and which CLI the engine actually runs +status: implemented +date: 2026-11-02 +scope: hookprobe +--- + +## Decision + +The investigator hands the SDK its message with `verbatim_prompts=True`. The +message is an alert's text, a reply under a card, or a watch round: written by +whoever can reach those doors, never by an operator at the CLI's prompt. Taken +as typed input, two things in it acted before any tool call: + +- an `@path` anywhere in the text made the CLI read that file and give it to + the model as a Read result that no PreToolUse hook saw, so neither the input + guard nor the audit knew a read had happened; +- a message starting with `/` ran a CLI command instead of an investigation: + `/cost show me` came back as the run's report, a cost table. + +Measured with the real service in its image, a recording stand-in for the model +API on an internal Docker network, and a marker file mounted into the +container. A request naming the marker with `@` put the marker's content into +the request the stand-in received, on the published 0.9.2 image and on this +change's image before the switch. After the switch the content never arrived, +`/cost show me` reached the model as text, and the scripted turn still ran: an +allowed read executed, and a `touch` into `results/` was refused and left no +file. + +The option arrived in claude-agent-sdk 0.2.158 and needs CLI 2.1.248 or newer. +It came with the relock of Dependabot's first proposal for the investigator's +runtime, which moved the SDK from 0.2.140 to 0.2.176. + +## Why + +The guard and the audit are built on tool calls. A read that happens before any +tool call is outside both, so the only fix that holds is to stop the CLI from +reading on the text's say-so. The SDK ships that switch for exactly this case. +A filter of our own over the text would have to know every way the CLI +interprets input, and that changes with every CLI release. + +Nothing relied on the expansion. No message the service builds carries a slash +command or an `@` mention, and skills load through setting sources, not slash +commands. + +## Consequences + +- A person who writes `@/some/file` in a chat reply no longer gets the file + attached. The model sees the words and may read the file itself, as a tool + call the guard checks and the audit records. +- A slash command in a reply is text for the model, not a CLI command. +- `tests/test_engine_loop.py` pins the option, and + `scripts/assert_guards_are_tested.py` flips it and requires that test to fail. +- Not measured: the codex and pi runtimes, which hand the message to their own + CLIs. +- Found on the way and not decided here: the engine runs the CLI bundled inside + the SDK wheel, not the one the Dockerfile installs from npm with + `CLAUDE_CODE_VERSION`. The SDK looks for its bundled binary first. The 0.9.2 + image ran 2.1.235 with 2.1.259 unused on PATH, and this relock moves the + engine to 2.1.316. So the lock, not the build argument, has been pinning the + agent runtime since the SDK began bundling it. Whether the image should drop + the npm CLI or point the engine at it is the operator's call. diff --git a/docs/containment.md b/docs/containment.md index d868a1ef..7d424fa1 100644 --- a/docs/containment.md +++ b/docs/containment.md @@ -146,7 +146,7 @@ Every row here is a claim, and a claim is worth what its test is worth. `scripts | **Chained audit** — every flight-recorder line carries the hash of the one before it; `verify_chain` names the first line that stops adding up, and `/v1/selftest` walks it | A record quietly tidied after the fact. The audit is what an investigation is reconstructed from and what a compliance reader asks about, and an append-only file cannot tell a line rewritten later from one written that way. Edits, deletions and reorderings are all caught, because each line names its predecessor rather than only hashing itself | Somebody who can rewrite the audit here AND reach the pipe's ledger. Rewriting `.chain` alone is not enough since 2026-09-10: every report carries `meta.audit_head` off-box, so a rebuilt chain is locally consistent and still fails `chain_anchored` against a head another service recorded hours earlier. Two disks and two services is the bound, not proof. Lines from before the chain existed are counted, never treated as a break — an alarm that fired on all existing history would be ignored by day two | `hookprobe/gate.py` (`append_audit`, `verify_chain`), `selftest.py` | | **What an approved procedure may run, and with what** — `HOOKPROBE_REMEDIATION_ALLOWLIST` (full-match regexes, one per line, hot-read; no file means nothing runs), checked over the WHOLE procedure at approval and again before each step; `execution_env()` passes `PATH`/`HOME`/locale, the cloud credentials an operator mounted for the purpose (`AWS_*`, `KUBE*`, `GOOGLE_*`, `AZURE_*`) and the egress proxy variables, and nothing else; a WRITING node names what its credential may do (`HOOKPROBE_BLAST_RADIUS`, one measured line per entry as `/v1/posture` reports it) and under `enforce` a credential that can do more refuses to start, naming the excess | The one path that executes anything running a command nobody wrote down, or a half-executed 1-2-3 when the file is tightened mid-incident; a procedure reading the family's HMAC keys, the Lark app secret or the provider credential out of its own environment — until 2026-09-10 `create_subprocess_exec` passed no `env` at all; a writing node quietly growing, which under `danger-only` had nothing to be compared against for its whole life | A pattern an operator wrote too widely — the file is the decision, this only holds it, and the model proposing anything stays free. A permitted command doing something with the credentials it IS given. Anything inside the declared radius, a grant that was unwise (`iam:*` declared is `iam:*`), and a credential widened while the node runs, which is caught at the next boot and by the hourly selftest, not at the moment it changes | `hookprobe/remediation.py` (`deny_reason`, `approve`, `execute`, `execution_env`), `posture.py` (`declared_radius`, `beyond`, `verdict`) | | **Approval window, freshness cursor and the version pressed** — a proposal is unrunnable 24h after it was made (the span the pipe gives a card's token), and it records what was true about its condition when the steps were chosen — has it ENDED, has the investigation taken another turn; if either has moved by the time somebody presses, the procedure is retired instead of run, the button is not drawn, and the refusal returns to the chat as a report. And a press names WHICH proposal it approves: the digest of its steps, run, clock and cursor as they were shown — inside the card's signed token, or as the console listed them — and a press naming none or another version is refused with the row untouched; the approved digest goes on the row | Commands chosen from evidence that has since moved being run by a click on a week-old card; approving a fix for a condition that already cleared, or running steps lifted from a report the investigation has since superseded — both reachable at any age under a clock alone, because a clock assumes the world moves at a rate; and commands nobody saw running under an approval of others — a proposal rewritten under the same id between the reading and the press | An approval at 23 hours on evidence that went stale in one is what the cursor is for; what neither sees is anything that happened outside this pipe. Nothing here reaches out to look, and it must not — the production investigator holds no credentials for the systems it writes procedures about — so somebody fixing the target by hand without saying so in chat is invisible | `hookprobe/remediation.py` (`APPROVAL_WINDOW_SECONDS`, `stale`, `cursor`, `moved`, `content_hash`, `Changed`), `actions.py`, `events.py`, `service.py` (`report_superseded`) | -| **Bash guard and input guard** — under `readonly`, mutating verbs of known CLIs with `aws` inverted to deny-unless-read, under `danger-only` only what no credential scope can undo; and writes to the files that steer the next run (`CLAUDE.md` and `AGENTS.md`, `.claude/`, the system prompt, `audit/`, `telemetry/`, `remediation/`, `actions/`, and the `results/` case files the prompt tells every run to read first) refused in the three shapes a write arrives in — an edit tool's path argument, the paths inside a codex `apply_patch` envelope, and a shell command that redirects, `tee`s, `sed -i`s, `cp`s, `mv`s, removes (`rm`, `find -delete`) or links (`ln`) into one of them; and under every posture, a script piped or handed to a shell (`curl … \| sh`), whose verbs no rule ever reads | An over-eager model running `kubectl delete` or `aws s3 rm`, or an installer's `curl … \| sh`; a run installing its own runbook as a persistence vector, or deleting the case files the next run reads. All three write shapes were measured getting through before they were closed: `apply_patch` because the guard read argument keys and codex puts its paths inside a patch string, `AGENTS.md` because the list only knew Claude's name for the steering file, and the shell redirect on **every** runtime under **both** postures; removing and symlinking from the shell were measured getting through on 2026-10-27 | An adversary: the rules need a binary and a verb in one pipeline segment, and separating them gets through; a redirect assembled at runtime out of two variables, or any of the endless other ways a shell can name a file; a script saved to a file and then run. These catch the obvious forms — the over-eager model, which is the case that happens. Writes anywhere else in the workspace are allowed on purpose. **The read-only credential is what holds** | `hookprobe/guard.py`, `inputs.py`, `gate.py` (`patch_targets`, `shell_write_target`) | +| **Bash guard and input guard** — under `readonly`, mutating verbs of known CLIs with `aws` inverted to deny-unless-read, under `danger-only` only what no credential scope can undo; and writes to the files that steer the next run (`CLAUDE.md` and `AGENTS.md`, `.claude/`, the system prompt, `audit/`, `telemetry/`, `remediation/`, `actions/`, and the `results/` case files the prompt tells every run to read first) refused in the three shapes a write arrives in — an edit tool's path argument, the paths inside a codex `apply_patch` envelope, and a shell command that redirects, `tee`s, `sed -i`s, `cp`s, `mv`s, removes (`rm`, `find -delete`) or links (`ln`) into one of them; and under every posture, a script piped or handed to a shell (`curl … \| sh`), whose verbs no rule ever reads; and before any tool runs, the message itself reaches the model as words, not as input typed at the CLI's prompt | An over-eager model running `kubectl delete` or `aws s3 rm`, or an installer's `curl … \| sh`; a run installing its own runbook as a persistence vector, or deleting the case files the next run reads. All three write shapes were measured getting through before they were closed: `apply_patch` because the guard read argument keys and codex puts its paths inside a patch string, `AGENTS.md` because the list only knew Claude's name for the steering file, and the shell redirect on **every** runtime under **both** postures; removing and symlinking from the shell were measured getting through on 2026-10-27; an `@path` in an alert or a chat reply making the CLI read that file into the model's context as a Read result no hook saw, and a leading `/` running a CLI command in place of the investigation, both measured getting through on 2026-11-02 | An adversary: the rules need a binary and a verb in one pipeline segment, and separating them gets through; a redirect assembled at runtime out of two variables, or any of the endless other ways a shell can name a file; a script saved to a file and then run. These catch the obvious forms — the over-eager model, which is the case that happens. Writes anywhere else in the workspace are allowed on purpose. The model can still read a file the text names, on its own initiative; that read is a tool call the audit records. **The read-only credential is what holds** | `hookprobe/guard.py`, `inputs.py`, `gate.py` (`patch_targets`, `shell_write_target`), `engine.py` (`verbatim_prompts`) | | **Memory shape check** — a suggested fact whose SHAPE could act waits for a human | An instruction smuggled through alert text into `CLAUDE.md` | A benign-shaped line that is simply wrong | `hookprobe/suggestions.py` | | **High-risk allowlist and target cooldown** — a step the report marked `risk: high` must also full-match a pattern in `HOOKPROBE_REMEDIATION_HIGH_RISK_ALLOWLIST`, deny-by-default in the same direction, both lists re-read before each step and one uncovered high step refusing the whole procedure at the click; after a procedure acts, a second one sharing its declared `target` **or** its literal command is refused for 15 minutes (`HOOKPROBE_REMEDIATION_COOLDOWN_SECONDS`, 0 disables), and one already `running` holds its target with no window at all | Arming remediation from also arming its worst half by the same gesture — `kubectl rollout restart .*` written to cover restarts has not licensed the steps the model itself calls dangerous; a fix and the rollback of that fix approved a minute apart, and two sequences interleaving their steps against one host. Held, not retired: the row stays `proposed` and the same press works after the window | **Keys the model writes.** A dangerous command the report labelled `low` never meets the strict list — the risk label is the MODEL's, so this can only ADD a requirement, and a mislabelled step is bounded by the ordinary allowlist exactly as before. A step that varies its command by a flag AND labels its target differently is not cooled; two keys because production named one target three ways while running character-identical commands, so the command half is the one that holds. Both count only what THIS node ran — somebody fixing the same box by hand is invisible, the same blind spot the freshness cursor has | `hookprobe/remediation.py` (`step_deny_reason`, `cooldown_keys`, `cooling`), `actions.py`, `service.py` (`report_cooling`) | | **Refused at config load** — `on_error` per stage (`pass` or `drop`, chosen per stage, never defaulted globally); an inline decider's timeout capped at 10s; no unreachable door, starved exit, or wildcard a return door can reach; a write-back channel may say `POST`, `PUT` or `PATCH` and nothing else | A stage's failure silently changing policy; an inline decider holding the sender's connection until it retries and duplicates the alert; a config change silently feeding a brain its own output; a channel configured with `DELETE`, or a `GET` whose side effect fires when a chat client fetches a link preview — the trap `/card-action` refuses to fall into | Choosing wrong inside what is legal: an explicit fail-open, a node that answers in 9s and is still the wrong place for the work, a graph that is legal and still wrong, a `POST` that does damage at the receiver — the pipe checks the verb it is asked for, never what the far side does with it | `hookrelay/processors.py` (`MAX_INLINE_TIMEOUT_SECONDS`), `config.py` (`_WRITE_METHODS`), `topology.py`, `scripts/assert_topology.py` | diff --git a/hookprobe/hookprobe/engine.py b/hookprobe/hookprobe/engine.py index 02a9f5fd..8eb5ba1a 100644 --- a/hookprobe/hookprobe/engine.py +++ b/hookprobe/hookprobe/engine.py @@ -1047,6 +1047,18 @@ async def _step_done(input_data: dict[str, Any], tool_use_id: str | None, contex # console is for, and an agent that says nothing for two minutes and # then everything at once is indistinguishable from a hung one. include_partial_messages=True, + # The message is not someone typing at the CLI's prompt: it is an + # alert's text, a reply under a card, a watch round, written by + # whoever can reach those doors. Delivered as typed input, an + # `@path` anywhere in it made the CLI read that file and hand it to + # the model as a Read result no PreToolUse hook saw, so neither the + # guard nor the audit knew the read happened; and a message that + # starts with `/` ran a CLI command instead of an investigation + # (`/cost` came back as the report). Both were measured on the + # 0.9.2 image against a recording stand-in model. Verbatim, the + # words arrive as words; the model can still ask to read a file, + # and that read is a tool call the guard and the audit see. + verbatim_prompts=True, # The declared MCP tools ride along so the CLI exposes them at all. # This is visibility, not enforcement — with bypassPermissions an # allowlist is a preference. _mcp_guard_hook is the gate. diff --git a/hookprobe/tests/test_engine_loop.py b/hookprobe/tests/test_engine_loop.py index 44a08d97..1f21fdb9 100644 --- a/hookprobe/tests/test_engine_loop.py +++ b/hookprobe/tests/test_engine_loop.py @@ -74,9 +74,11 @@ class _FakeClient: connected = 0 disconnected = 0 queries: list[str] = [] + options_seen: Any = None def __init__(self, options: Any = None) -> None: self.options = options + type(self).options_seen = options async def connect(self) -> None: type(self).connected += 1 @@ -102,6 +104,7 @@ def engine(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): _FakeClient.disconnected = 0 _FakeClient.queries = [] _FakeClient.scripted = [] + _FakeClient.options_seen = None monkeypatch.setattr(claude_agent_sdk, "ClaudeSDKClient", _FakeClient) settings = make_settings(tmp_path, workdir=tmp_path) return ClaudeAgentEngine(settings) @@ -112,6 +115,19 @@ def _run(engine: ClaudeAgentEngine, messages: list[Any]): return asyncio.run(engine.run(message="investigate", session_key="probe:t:1")) +def test_the_message_reaches_the_model_as_words_not_as_cli_input(engine) -> None: + """Alert text, a reply under a card, a watch round: none of it is the + operator typing at the CLI's prompt. As typed input, an `@path` in it made + the CLI read that file into the model's context with no tool call for the + guard to refuse or the audit to record, and a leading `/` ran a CLI command + in place of the investigation. Measured on the 0.9.2 image; verbatim + prompts deliver the words as words.""" + _run(engine, [_result()]) + + assert _FakeClient.options_seen is not None + assert _FakeClient.options_seen.verbatim_prompts is True + + def test_deltas_are_streamed_but_not_counted(engine) -> None: """The 32,294 bug, as an assertion. diff --git a/scripts/assert_guards_are_tested.py b/scripts/assert_guards_are_tested.py index 35ef0665..e630ca7f 100755 --- a/scripts/assert_guards_are_tested.py +++ b/scripts/assert_guards_are_tested.py @@ -20,7 +20,7 @@ python3 scripts/assert_guards_are_tested.py # every mutation python3 scripts/assert_guards_are_tested.py --list # names only -WHAT THIS IS NOT. It is not mutation coverage. Seven modules and a hand-written +WHAT THIS IS NOT. It is not mutation coverage. A few modules and a hand-written list is not a proof about the suite; it is a spot check on the claims that would hurt most if their tests were decorative. Add one whenever a boundary is added, which is the whole ceremony. @@ -41,6 +41,13 @@ # (name, file, exact text to replace, replacement, tests that must then fail) MUTATIONS: tuple[tuple[str, str, str, str, tuple[str, ...]], ...] = ( + ( + "alert and chat text reach the CLI as typed input again", + "hookprobe/hookprobe/engine.py", + " verbatim_prompts=True,", + " verbatim_prompts=False,", + ("tests/test_engine_loop.py",), + ), ( "the read-only guard allows everything", "hookprobe/hookprobe/guard.py",