feat(benchmark): add an independent native Claude Code runner

benchmark/README.md

@@ -62,6 +62,9 @@ helper delivery neither launches trials nor changes frozen historical arms.
   patch and synthetic HTTP/client/Docker checks for the frozen single-host v1
   sources. It preserves historical attribution and does not launch a model or
   native-grader trial.
+- [Native Claude Code](runtime/NATIVE_CLAUDE.md) provides an independent,
+  explicitly pinned SForge worker with native Stop hooks and session recovery.
+  Model, tool, Goal/Plan and submission qualification precedes long trials.
 - [Execution observations through ordinary Goal planning](runtime/EXECUTION_OBSERVATIONS.md)
   covers independent local measurements with ordinary Goal/Todo planning and
   stride selection, complete initial Goal retention, and migration from the

benchmark/edgebench/claude_run.py

@@ -0,0 +1,95 @@
+"""Run an independent native Claude trial with the selected SForge provider.
+
+The judge must already be running. This entrypoint neither schedules a cohort
+nor changes another experiment's sources, credentials, containers or board.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+
+from benchmark.edgebench.source import active_source_receipt, bootstrap
+from benchmark.runtime.source import checked_revision
+
+if __name__ == "__main__":
+    bootstrap()
+
+
+def main(argv=None):
+    from sforge.harness.benchmark import load_benchmark
+    from sforge.harness.backend.docker_backend import DockerBackend
+    from sforge.harness.config import SForgeConfig
+    from sforge.harness.run_agent import run_agent
+    from sforge.harness.task_spec import make_task_spec
+    from benchmark.runtime.sforge_claude import (
+        CLAUDE_EFFORTS, NativeClaudeWorker,
+    )
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--task", required=True)
+    parser.add_argument("--tasks-dir", type=Path, required=True)
+    parser.add_argument("--log-dir", type=Path, required=True)
+    parser.add_argument("--run-id", required=True)
+    parser.add_argument("--runner-revision", required=True,
+                        help="Expected clean research-runner commit")
+    parser.add_argument("--judge-url", required=True)
+    parser.add_argument("--claude-binary", type=Path, required=True)
+    parser.add_argument("--claude-sha256", required=True)
+    parser.add_argument("--claude-version", required=True)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--effort", choices=CLAUDE_EFFORTS, required=True)
+    parser.add_argument("--timeout", type=int, required=True)
+    parser.add_argument("--submission-cooldown", type=int, required=True)
+    parser.add_argument("--worker-cpus", type=int, default=2)
+    parser.add_argument("--worker-memory", default="4g")
+    parser.add_argument("--judge-cpus", type=int, default=1)
+    parser.add_argument("--judge-memory", default="2g")
+    args = parser.parse_args(argv)
+    if any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", value)
+           for value in (args.task, args.run_id)):
+        parser.error("task and run identifiers must be simple names")
+    if args.timeout <= 0 or args.submission_cooldown < 0:
+        parser.error("timeout must be positive and submission cooldown non-negative")
+    if min(args.worker_cpus, args.judge_cpus) <= 0:
+        parser.error("resource limits must be positive")
+    runner_revision = checked_revision(Path(__file__).resolve().parents[2], args.runner_revision)
+    config = SForgeConfig(
+        agent_api_base_url=os.environ.get("ANTHROPIC_BASE_URL"),
+        agent_api_key=os.environ.get("ANTHROPIC_AUTH_TOKEN") or os.environ.get("ANTHROPIC_API_KEY"),
+        agent_model=args.model, agent_effort=args.effort, agent_timeout=args.timeout,
+        log_dir=args.log_dir, tasks_dir=args.tasks_dir,
+        work_cpu_limit=args.worker_cpus, work_mem_limit=args.worker_memory,
+        judge_cpu_limit=args.judge_cpus, judge_mem_limit=args.judge_memory,
+    )
+    agent = NativeClaudeWorker(config, binary=args.claude_binary,
+                              sha256=args.claude_sha256, version=args.claude_version)
+    task = make_task_spec(args.tasks_dir / f"{args.task}.json", load_benchmark(args.tasks_dir))
+    if task.game_mode:
+        parser.error("This entrypoint qualifies file-submission tasks only")
+    os.umask(0o077)
+    # Admission, total deadline, submission limits, scoring, auto-resume and
+    # cleanup are native provider responsibilities. No extra model loop here.
+    result = run_agent(task_spec=task, agent=agent, config=config, backend=DockerBackend(),
+        run_id=args.run_id, model=args.model, timeout=args.timeout,
+        judge_url=args.judge_url, eval_interval=0, disable_auto_eval=True,
+        disable_stop_hook=False, disable_auto_resume=False,
+        submission_cooldown=args.submission_cooldown, internet=task.internet)
+    receipt = {"agent": agent.name, "model": args.model, "effort": args.effort,
+               "claude_version": args.claude_version, "binary_sha256": args.claude_sha256,
+               "runner_revision": runner_revision,
+               "provider": active_source_receipt(), "result": result.to_dict()}
+    target = args.log_dir / "runs" / args.run_id / task.task_id / "claude-result.json"
+    target.write_text(json.dumps(receipt, indent=2) + "\n")
+    print(json.dumps(receipt))
+    # SForge preserves structured execution evidence even when it catches an
+    # exception. Do not turn a swallowed launch failure into CLI success.
+    execution = result.solver_execution or {}
+    return 0 if (result.runtime_seconds > 0 and result.grading_status == "complete"
+                 and execution.get("status") in {"normal_exit", "timed_out"}) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmark/runtime/NATIVE_CLAUDE.md

@@ -0,0 +1,71 @@
+# Native Claude Code research runner
+
+`benchmark.edgebench.claude_run` runs an independent file-submission trial using
+SForge's `ClaudeCodeAgent`. Claude Code owns tools, sessions and Stop hooks;
+SForge owns submission, grading, the total deadline and `--continue` recovery.
+There is no LoopX Goal/Todo runtime or implicit relationship to another arm.
+The ordinary benchmark runner and existing experiments are unchanged.
+
+The transport installs an operator-supplied native Claude binary with an explicit
+SHA-256 and version. It reuses SForge's native Stop hook. Both that script and
+`sforge-submit` receive explicit agent-readable executable modes, even when the
+operator runs with umask 077. The currently pinned SForge task installer makes
+generated, non-secret task files readable with mode 0644, including when host
+and container UIDs differ; unrelated uploads retain their original permissions.
+The runner uses that provider owner and its ordinary Docker backend. It does
+not promise that the task prompt is private between container users. The older
+prompt-only copy override has been removed because the provider's subsequent
+task-file installation superseded its 0600 setting. This follows the current
+provider contract without broadening general upload permissions.
+Python belongs here as an existing provider
+transport, not a second control-plane decision owner.
+
+Select a clean runner revision and freeze the SForge source with the existing
+source-admission workflow. Prepare an isolated judge, task directory, log
+directory, run identity and resource admission before invoking the runner.
+Verify judge connectivity from both the host and container: SForge maps
+`host.docker.internal` to loopback for host registration. A judge listening only
+on the Docker bridge therefore needs its reachable bridge address as the URL:
+
+```bash
+# Supply ANTHROPIC_BASE_URL and ANTHROPIC_AUTH_TOKEN through private environment
+# configuration. The endpoint must implement Anthropic Messages for the model.
+python -m benchmark.edgebench.claude_run \
+  --tasks-dir "$TASKS" --log-dir "$LOGS" --task "$TASK" --run-id "$RUN" \
+  --runner-revision "$RUNNER_SHA" \
+  --sforge-revision "$PROVIDER_SHA" --judge-url "$JUDGE_URL" \
+  --claude-binary "$CLAUDE_BINARY" --claude-sha256 "$CLAUDE_SHA256" \
+  --claude-version "$CLAUDE_VERSION" --model "$MODEL" --effort max \
+  --timeout 21600 --submission-cooldown 1800
+```
+
+The default worker resources are 2 CPUs / 4 GiB; judge resources are 1 CPU /
+2 GiB. These limits do not establish host capacity or authorize oversubscription.
+The command does not start a judge, launch a cohort or mutate another run.
+The provider's native Stop hook continually requests work until its outer
+deadline; do not install it in ordinary personal Claude settings. Disable this
+experiment by stopping its own launcher, using the provider's normal cleanup.
+
+An Anthropic-compatible gateway is required for a non-Claude model. A model name
+and successful HTTP response alone do not qualify compatibility. Before a long
+run, observe the outgoing model and effort, verify streamed tool calls/results,
+Stop continuation, exact-session resume and a real native submission/result.
+Keep raw trajectories, credentials and synthetic run evidence outside Git.
+
+Claude's native `/goal` and Plan mode can be tested separately with the same
+endpoint. `/goal` adds a model-evaluated completion condition; Plan mode restricts
+editing. This runner does not silently enable either. The adapter pins the
+small-model aliases, including the Goal evaluator, to the selected model, and
+passes the requested effort on both initial and resumed invocations. Provider
+translation and usage/cache accounting still need live verification.
+
+Validation uses `python -m pytest benchmark/tests/test_native_claude.py` with
+the selected SForge source on `PYTHONPATH`. Standard test collection skips this
+optional module when SForge is absent. Set
+`LOOPX_EDGEBENCH_DOCKER_SMOKE=1` for the disposable Docker upload/permission
+case (requires a local `python:3.12-slim` image). It exercises the provider's
+complete task-file installation, including its final permission readback, and
+checks that unrelated private uploads remain restricted. That test uses a synthetic
+binary; real CLI/model, Goal/Plan, submission and stability qualification are
+separate, explicitly authorized operational checks. Short checks do not prove
+long-horizon reliability or benchmark improvement.

benchmark/runtime/sforge_claude.py

@@ -0,0 +1,93 @@
+"""Offline installation of the native SForge Claude Code agent.
+
+This Python transport keeps the provider's Stop hook, tool permissions and
+outer resume/deadline lifecycle. It does not install LoopX goal control.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path, PurePosixPath
+import re
+import shlex
+
+from sforge.harness.agent.claude_code import ClaudeCodeAgent
+
+# Native CLI vocabulary, local to this provider transport.
+CLAUDE_EFFORTS = ("low", "medium", "high", "xhigh", "max")
+
+
+class NativeClaudeWorker(ClaudeCodeAgent):
+    """An explicitly pinned native binary, installed without a package download."""
+
+    name = "native-claude-offline"
+    install_cmds = []
+
+    def __init__(self, config, *, binary: Path, sha256: str, version: str):
+        super().__init__(config)
+        if not config.agent_model or config.agent_effort not in CLAUDE_EFFORTS:
+            raise ValueError("Explicit model and supported native Claude effort are required")
+        if not config.agent_api_base_url or not config.agent_api_key:
+            raise ValueError("An explicit Anthropic-compatible endpoint and credential are required")
+        if not re.fullmatch(r"[0-9a-f]{64}", sha256):
+            raise ValueError("Expected binary SHA-256 is required")
+        if not re.fullmatch(r"\d+\.\d+\.\d+", version):
+            raise ValueError("Expected native Claude version is required")
+        self.binary = Path(binary).resolve(strict=True)
+        self.sha256, self.version = sha256, version
+        self._verify_binary()
+
+    def _verify_binary(self):
+        with self.binary.open("rb") as stream:
+            actual = hashlib.file_digest(stream, "sha256").hexdigest()
+        if actual != self.sha256:
+            raise ValueError("Native Claude binary differs from its declared digest")
+
+    def augment_env(self, env, model):
+        super().augment_env(env, model)
+        selected = model or self._config.agent_model
+        # Include the native /goal evaluator and other small-model calls in
+        # the declared model budget; do not silently use another model family.
+        for key in ("ANTHROPIC_DEFAULT_HAIKU_MODEL", "ANTHROPIC_DEFAULT_SONNET_MODEL",
+                    "ANTHROPIC_DEFAULT_OPUS_MODEL", "ANTHROPIC_SMALL_FAST_MODEL",
+                    "CLAUDE_CODE_SUBAGENT_MODEL"):
+            env[key] = selected
+        env["CLAUDE_CODE_EFFORT_LEVEL"] = self._config.agent_effort
+        env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1"
+
+    def format_run_cmd(self, prompt_path, **kwargs):
+        # The same native flag applies on initial execution and --continue.
+        return (super().format_run_cmd(prompt_path, **kwargs)
+                + " --effort " + shlex.quote(self._config.agent_effort))
+
+    def install_stop_hook(self, backend, handle, log_dir, logger):
+        self._verify_binary()
+        backend.copy_to_container(handle, self.binary, PurePosixPath("/usr/local/bin/claude"))
+        result = backend.exec_run(handle, ["chmod", "0755", "/usr/local/bin/claude"], user="root")
+        if result.exit_code:
+            raise RuntimeError("Could not install native Claude executable")
+        digest = backend.exec_run(handle, ["sha256sum", "/usr/local/bin/claude"], user="agent")
+        if digest.exit_code or digest.output.split()[:1] != [self.sha256]:
+            raise RuntimeError("Installed native Claude binary digest differs")
+        observed = backend.exec_run(handle, ["/usr/local/bin/claude", "--version"], user="agent")
+        if observed.exit_code or observed.output.strip() != f"{self.version} (Claude Code)":
+            raise RuntimeError("Installed native Claude version differs from the declared version")
+        super().install_stop_hook(backend, handle, log_dir, logger)
+        # Native uploads preserve host umask. An executable root-owned shell
+        # script still needs agent read permission, including sforge-submit.
+        result = backend.exec_run(handle, ["/bin/sh", "-ec",
+            "chmod 0755 /tmp/sforge-stop-hook.sh /usr/local/bin/sforge-submit; "
+            "chmod 0600 /home/agent/.claude/settings.json"], user="root")
+        if result.exit_code:
+            raise RuntimeError("Could not set native hook/submission permissions")
+        result = backend.exec_run(handle, ["/bin/sh", "-ec",
+            "test -r /tmp/sforge-stop-hook.sh; test -x /tmp/sforge-stop-hook.sh; "
+            "test -r /usr/local/bin/sforge-submit; test -x /usr/local/bin/sforge-submit; "
+            "test -r /home/agent/.claude/settings.json"], user="agent")
+        if result.exit_code:
+            raise RuntimeError("Native hooks or submission entrypoint are unavailable to agent")
+        (Path(log_dir) / "claude-install.json").write_text(json.dumps({
+            "version": self.version, "binary_sha256": self.sha256,
+            "model": self._config.agent_model, "effort": self._config.agent_effort,
+            "stop_hook": "native", "resume": "native", "installation_verified": True,
+        }, indent=2) + "\n")