validate-harborize-instrument: free rungs 0-5 plus canary and mechanical packages (tasks 1-8)

.gitattributes

@@ -1,6 +1,7 @@
 flake.lock linguist-generated=true
 bun.lock linguist-generated=true
 machines/*/facter.json linguist-generated=true
+modules/home/ai/evals/harborize/*-harbor/** linguist-generated=true
 packages/docs/src/grammars/*.json linguist-vendored=true
 *.sh linguist-detectable=false
 scripts/** linguist-detectable=false

.gitignore

@@ -27,6 +27,10 @@ yarn-debug.log*
 yarn-error.log*
 pnpm-debug.log*
 
+# Python bytecode
+__pycache__/
+*.pyc
+
 # wrangler (Cloudflare Workers)
 .wrangler/
 .dev.vars*

modules/home/ai/evals/harborize/cells/cells-acp.json

@@ -0,0 +1 @@
+[{"name": "neg-acp", "runner": "harbor", "agent": "acp:claude-agent", "model": "anthropic/claude-opus-5"}]

modules/home/ai/evals/harborize/cells/cells-nonconsuming.json

@@ -0,0 +1 @@
+[{"name": "neg-aider", "runner": "harbor", "agent": "aider", "model": "anthropic/claude-opus-5"}]

modules/home/ai/evals/harborize/cells/cells.json

@@ -0,0 +1 @@
+[{"name": "codex-sol", "runner": "harbor", "agent": "codex", "model": "gpt-5.6-sol", "env": {"HARBOR_TELEMETRY": "0", "CODEX_FORCE_AUTH_JSON": "1"}}]

modules/home/ai/evals/harborize/checks/resolve_check.py

@@ -0,0 +1,33 @@
+"""Resolve a condition directory through the installed Harbor CLI's own skills API.
+
+Run with the interpreter that backs the `harbor` entrypoint rather than a bare
+`python3`, because the proposition this check witnesses is that the *installed*
+CLI resolves the directory:
+
+    "$(sed -n '1s|^#!||p' "$(command -v harbor)")" checks/resolve_check.py <dir>
+
+The house PEP-723 header is deliberately absent. `uv run --script` always builds
+an isolated environment, so a `dependencies = ["harbor"]` declaration would
+resolve a second copy of Harbor and stop measuring the CLI under test.
+"""
+
+import json
+import sys
+
+from harbor.skills import compute_skill_digest, resolve_skills
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print(f"usage: {sys.argv[0]} <condition-dir>", file=sys.stderr)
+        return 2
+    resolved = resolve_skills([sys.argv[1]])
+    print(json.dumps([
+        {"name": s.name, "source": str(s.source), "digest": compute_skill_digest(s.source)}
+        for s in resolved
+    ], indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

modules/home/ai/evals/harborize/conditions/canary/harborize-injection-canary/SKILL.md

@@ -0,0 +1,10 @@
+---
+name: harborize-injection-canary
+description: Injection canary for the harborize instrument. Carries a single token that a canary task's oracle extracts and its verifier asserts, so a failure to deliver injected skills fails a task rather than silently voiding a batch.
+---
+
+# Harborize injection canary
+
+This skill exists to be delivered, not to be used.
+
+The canary token is HARBORIZE-CANARY-9F3A21 and it appears nowhere else in the deployed tree.

modules/home/ai/evals/harborize/injection-canary/README.md

@@ -0,0 +1,50 @@
+# injection-canary
+
+Instrument-integrity canary for the harborize instrument.
+
+Instrument version: 0.2.1 (frozen for the duration of the `validate-harborize-instrument` change).
+
+Upstream pins every source anchor cites: harbor `ac398bbda7c4c1073461797d3b95c2455cc671b5`, benchflow `d30527b82027a416e72014920cdf43a534967ad3`, skillsbench `9a1f4dd5f7659f75707435da3ce854b6e48321d1`.
+
+## What it asserts
+
+The oracle greps every skill discovery root (`/harbor/skills`, the per-agent `.claude`/`.codex`/`.opencode`/`.agents` paths, and `/skills`) for the canary token and writes what it finds to `/logs/artifacts/canary-output.txt`; the verifier rewards 1 exactly when that file holds `HARBORIZE-CANARY-9F3A21`.
+
+The token exists in exactly two places: the canary `SKILL.md` this package injects, and the asserted literal in its own `verifier/test.sh`.
+It appears nowhere in the skill corpus under evaluation, which is what makes a reward of 1 evidence that delivered skill content reached a container discovery path.
+The verifier's copy does not weaken that inference, because no agent phase can observe it — both runners upload the verifier's directory during the verification phase, after the agent phase has ended (see the fork section).
+The claim this replaces, that the token lives only in the injected `SKILL.md`, was false in the other direction: the verifier's copy is real, and is why the leakage audit flags this package below.
+
+The evidence is one-directional and only the positive direction is read.
+Under the oracle, which greps every discovery root unconditionally, reward 1 and reward 0 are both informative, and the falsifiability control at task 7.3 establishes that the 0 is reachable.
+Under a model-driven agent only reward 1 carries delivery information, because a model that never greps scores 0 with the skill perfectly delivered.
+That is why rung 6 asserts the adapter's registration directory instead.
+
+## Verifier fork: shared
+
+No `verifier/Dockerfile` and no `verifier.sandbox_mode`, so the verifier runs in the agent sandbox.
+The fork is forced rather than preferred, and each runner forces it for its own reason.
+
+BenchFlow refuses to launch a task that declares separate rather than falling back to shared.
+`runtime_capabilities.py:186-192` raises an unsupported-feature issue — its reason string reads "separate verifier sandboxes are parsed but not executed", which is the wording of a refusal and not of a fallback — and `raise_for_task_runtime_support` is a fail-closed pre-launch gate (`sandbox/setup.py:676`, `:819-842`).
+
+Harbor's separate verifier empties `/logs/verifier` before the verifier runs (`trial.py:599`), through the same host bind it mounts at `:686-692`.
+`_run_shared_verifier` (`trial.py:536-567`) performs no wipe, and both packages are single-step, which is the precondition that claim needs: the multi-step path calls `_reset_shared_step_verifier_dirs` (`multi_step.py:202`, defined at `:338-342`) before every shared step verifier, so a package that ever grows `[[steps]]` loses that directory too. `SingleStepTrial.__init__` raises on a stepped task (`single_step.py:28-29`), so the two cannot be confused silently.
+
+None of that decides the deliverable channel, because `/logs/verifier` is not the channel this package uses and cannot be.
+BenchFlow clears the directory's contents on the agent container immediately before the verifier runs, unconditionally (`sandbox/lockdown.py:775-784` called from `harden_before_verify` at `:1205-1212`), so a `canary-output.txt` written there is gone before `grep` looks for it under BenchFlow and present under Harbor's shared fork — the runner-dependent silent 0 the corpus exists to prevent.
+The oracle therefore writes `/logs/artifacts/canary-output.txt`, which both runners bind for the whole trial and neither hardening step touches; the workspace README's channel section carries the anchors and the rung-4 evidence.
+
+## Known leakage flag
+
+`audit_leakage.py` check 1 flags the token (`MIN_LITERAL_LENGTH` at `:44`, `check_literals` at `:96`) because the asserted literal appears in both the verifier and the injected `SKILL.md`.
+That is by design: the canary's mechanism is its answer key.
+The instrument is frozen at 0.2.1 and is not edited to exempt it; the instrument-side question is deferred to the next revision.
+
+A consequence that was accepted and turns out not to arise: the fork being shared does not put the token within a running agent's reach.
+Both runners upload the verifier's own directory during the verification phase, after the agent phase has ended — Harbor at `verifier/verifier.py:147-153`, reached from `_run_shared_verifier`, with the phase order fixed at `trial/single_step.py:41` then `:52`; BenchFlow at `task/verifier_core.py:385` in `_verify_test_script` (`:346`), reached from `verify()` (`:260`).
+No agent phase observes `test.sh`.
+
+Rung 6 still scores on the adapter's registration directory rather than on the reward, for a reason that does not depend on that: a model-driven trial's reward conflates delivery with the model's own behaviour, since a model that never greps scores 0 with the skill perfectly delivered.
+The registration directory is the deterministic witness; the reward is not.
+The falsifiability control runs under the oracle, which greps skill directories and never reads the verifier.

modules/home/ai/evals/harborize/injection-canary/environment/Dockerfile

@@ -0,0 +1,3 @@
+FROM ubuntu:24.04@sha256:561618e2c15bf2397621dd04f96926663a3b5616c189cf7e38db7e82f5c538ea
+
+WORKDIR /root

modules/home/ai/evals/harborize/injection-canary/oracle/solve.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+out=/logs/artifacts/canary-output.txt
+mkdir -p /logs/artifacts
+: > "$out"
+# These roots are the oracle discovery-path union, not the adapter registration
+# destinations; no adapter registration runs on the oracle path.
+# HOME is defaulted because under set -u a container exporting none aborts here
+# before any output is written, which reads downstream as a false canary alarm.
+home=${HOME:-/root}
+for root in /harbor/skills "$home/.claude/skills" "$home/.codex/skills" \
+            "$home/.opencode/skills" "$home/.agents/skills" /skills; do
+  [ -d "$root" ] || continue
+  for f in "$root"/*/SKILL.md; do
+    [ -f "$f" ] || continue
+    grep -ho 'HARBORIZE-CANARY-[A-Z0-9]*' "$f" >> "$out" || true
+  done
+done
+sort -u -o "$out" "$out"

modules/home/ai/evals/harborize/injection-canary/task.md

@@ -0,0 +1,38 @@
+---
+schema_version: '1.3'
+metadata:
+  author_name: vanixiets harborize validation
+  difficulty: easy
+  category: software-engineering
+  subcategory: skill-injection-canary
+  category_confidence: high
+  task_type:
+  - verification
+  modality:
+  - text
+  interface:
+  - terminal
+  skill_type:
+  - instrument-integrity
+  tags:
+  - canary
+  - skill-injection
+verifier:
+  type: test-script
+  timeout_sec: 300.0
+agent:
+  timeout_sec: 300.0
+  network_mode: no-network
+sandbox:
+  network_mode: public
+  build_timeout_sec: 600.0
+  os: linux
+  cpus: 1
+  memory_mb: 2048
+  storage_mb: 5120
+  gpus: 0
+---
+
+Find the canary token and make it the exact content of `/logs/artifacts/canary-output.txt`, as a single line, deduplicated and sorted, with no other output in that file.
+
+The token is not in this instruction.

modules/home/ai/evals/harborize/injection-canary/verifier/test.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -euo pipefail
+mkdir -p /logs/verifier
+if grep -qx 'HARBORIZE-CANARY-9F3A21' /logs/artifacts/canary-output.txt; then
+  echo 1 > /logs/verifier/reward.txt
+else
+  echo 0 > /logs/verifier/reward.txt
+fi