From aabce4f58198edb2ec37d021184de823a0f00ef4 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:56:54 +0800 Subject: [PATCH] fix(benchmark): retain research outside native submission trees Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- benchmark/edgebench/README.md | 22 ++++ benchmark/edgebench/prompts.py | 49 +++----- benchmark/edgebench/run.py | 27 ++-- benchmark/runtime/research_artifacts.py | 66 ++++++++++ benchmark/runtime/sforge.py | 51 +++++++- benchmark/runtime/sforge_backend.py | 13 +- benchmark/runtime/sforge_warm_start.py | 15 ++- benchmark/tests/test_edgebench_feedback.py | 2 + .../tests/test_edgebench_feedback_docker.py | 84 ++++++++++++- benchmark/tests/test_edgebench_prompts.py | 24 ++-- benchmark/tests/test_research_artifacts.py | 117 ++++++++++++++++++ benchmark/tests/test_sforge_runtime.py | 22 ++-- benchmark/tests/test_sforge_warm_start.py | 53 +++++++- 13 files changed, 467 insertions(+), 78 deletions(-) create mode 100644 benchmark/runtime/research_artifacts.py create mode 100644 benchmark/tests/test_research_artifacts.py diff --git a/benchmark/edgebench/README.md b/benchmark/edgebench/README.md index 551032bacc..9573f37cb1 100644 --- a/benchmark/edgebench/README.md +++ b/benchmark/edgebench/README.md @@ -671,6 +671,28 @@ and credentials belong outside the public repository. ## Research checkpoint storage +New native non-game runs give every worker profile the same research directory, +`/tmp/loopx-benchmark-research`, outside the task workspace and submitted paths. +Keep experiment notes, local probes and candidate history there; required build +and runtime inputs still belong in the task's declared submission paths. The +runner does not filter submissions, extend evaluator feedback or alter grading. +Overlapping submission/capture roots and existing unrelated data at this reserved +path reject setup before solver execution. + +Terminal collection retains this directory under `collected/research`. With an +explicit checkpoint request, the runner adds a separate `benchmark_research` +component to the staged declaration, without modifying the caller's file. Warm +continuation restores that exact component at its original path through the +ordinary verified research-file transport. Other external research components +still require explicit selection. Old checkpoints remain valid and preserve +their workspace files; they start with an empty external research directory, +without automatic relocation or deletion of historical artifacts. A warm receiver +appends a bounded storage-policy update to its existing `AGENTS.md`, preserving +the original task text and prior feedback policy; it does not rewrite the sealed +checkpoint or Goal state. Terminal +collection alone does not make a run resumable. Game and restricted-feedback +runs retain their existing storage behavior. + Full controller capture requires a filesystem that preserves case-sensitive and Unicode-normalization-sensitive filenames. A Linux controller using a shared host volume may not have those properties. New controller-enabled launches probe diff --git a/benchmark/edgebench/prompts.py b/benchmark/edgebench/prompts.py index 6c6ea31332..372403bcd5 100644 --- a/benchmark/edgebench/prompts.py +++ b/benchmark/edgebench/prompts.py @@ -6,8 +6,6 @@ from __future__ import annotations -from pathlib import PurePosixPath - def native_research_guidance() -> str: """Keep the task's optimization objective explicit.""" @@ -19,38 +17,31 @@ def native_research_guidance() -> str: ) -def checkpoint_research_instructions(request: dict, workspace: str) -> str: - """Describe declared coverage without inferring or expanding capture authority. +def research_artifact_instructions(*, checkpoint: bool) -> str: + """Separate reproducibility data from candidate inputs for every native arm.""" + from benchmark.runtime.research_artifacts import RESEARCH_DIRECTORY - Directory membership remains the artifact owner's responsibility at capture. - This provider guidance keeps newly generated scientific outputs in that scope; - it neither proves dependency closure nor changes the evaluator's file set. - """ - root = PurePosixPath(workspace) - covered = any( - isinstance(entry, dict) and isinstance(entry.get("source_path"), str) - and root.is_relative_to(PurePosixPath(entry["source_path"])) - for entry in request["components"] - ) - coverage = ( - "The checkpoint declaration includes this task workspace. New files saved " - "inside that directory are included when it is captured. " - if covered else - "The checkpoint declaration does not cover the whole task workspace. " - "Do not assume new research files are captured; report the missing directory " - "coverage before relying on continuation. " + capture = ( + "Checkpoint capture includes this directory as a separate component; " + "verified continuation restores its files at the same path. " + if checkpoint else + "Terminal artifact collection retains this directory for analysis; " + "that alone does not provide a resumable checkpoint. " ) return ( - "### Durable Research Artifacts\n\n" - "Checkpoint capture is enabled. " + coverage + - "Keep reusable research artifacts in a dedicated subdirectory of the task " - "workspace: candidate snapshots, fitted parameters, local test inputs, " + "### Research Artifacts\n\n" + f"Use `{RESEARCH_DIRECTORY}` for reusable research artifacts that are not " + "candidate inputs: candidate snapshots, fitted parameters, local test inputs, " "experiment reports and negative results needed to reproduce or reconsider " - "a decision. Use workspace-relative references where possible. Reserve " - "temporary directories for disposable scratch; a report that only names an " + "a decision. This directory is outside the task workspace and submitted paths. " + "Keep required candidate code and runtime/build data in the task's declared " + "submitted paths; the judge cannot use this research directory as an extra input. " + "Do not link it into the submission tree. " + capture + + "Use explicit references to retained artifacts. A report that only names an " "external file does not preserve its bytes. Keep existing external dependencies " - "explicit until they have been copied and their references updated and checked; " - "do not delete or silently substitute historical evidence.\n\n" + "explicit until they have been copied and their references updated and checked. " + "Older checkpoints preserve their original workspace files; this policy does " + "not relocate or discard historical evidence.\n\n" "This storage guidance does not change submitted files, task acceptance, " "feedback access or execution authority. Capture and receiver verification " "still determine which artifacts are actually available.\n\n" diff --git a/benchmark/edgebench/run.py b/benchmark/edgebench/run.py index a1feda16be..ca30469be8 100644 --- a/benchmark/edgebench/run.py +++ b/benchmark/edgebench/run.py @@ -39,8 +39,8 @@ from benchmark.runtime.sforge_backend import RecordingDockerBackend from benchmark.runtime.source import archive_source, source_pins from benchmark.edgebench.prompts import ( - blind_task_prompt, best_only_task_prompt, native_task_prompt, native_task_instructions, - checkpoint_research_instructions, + blind_task_prompt, best_only_task_prompt, native_task_instructions, + research_artifact_instructions, ) from benchmark.edgebench.online_sampling import OnlineSampler from benchmark.edgebench.feedback import BestOnlyFeedback, FEEDBACK_MODES, validate_best_only @@ -277,6 +277,13 @@ def main(argv=None): archive_source(runner, pins[1], Path(directory) / "runner.tar", ("benchmark/runtime",)) task_file = args.tasks_dir / f"{args.task}.json" task = make_task_spec(task_file, load_benchmark(args.tasks_dir)) + research_sidecar = args.feedback == "native" and not task.game_mode + if research_sidecar: + from benchmark.runtime.research_artifacts import validate_research_scope, with_research_capture + validate_research_scope(task.cwd, task.submit_paths) + if checkpoint_request is not None: + declaration = with_research_capture(declaration) + checkpoint_request = json.dumps(declaration).encode("utf-8") blind_endpoint = None feedback_prompt = None workspace_instructions = None @@ -300,14 +307,11 @@ def main(argv=None): native_policy = dict(submission_cooldown=args.submission_cooldown, eval_interval=args.eval_interval, internet=task.internet, selection=task.judge.selection, score_direction=task.judge.score_direction) - feedback_prompt = native_task_prompt(task.work.agent_query, task.submit_paths, **native_policy) workspace_instructions = native_task_instructions(task.submit_paths, **native_policy) - if checkpoint_request is not None: - research = checkpoint_research_instructions(declaration, task.cwd) - workspace_instructions += research - # Keep the exact policy prefix shared with AGENTS.md; the original - # task remains the separately owned Goal objective. - feedback_prompt = workspace_instructions + f"---\n\n{task.work.agent_query}\n" + workspace_instructions += research_artifact_instructions(checkpoint=checkpoint_request is not None) + # All native arms get the same storage policy. Checkpoint coverage is + # explicit; neither collection nor a sidecar changes native submissions. + feedback_prompt = workspace_instructions + f"---\n\n{task.work.agent_query}\n" task_objective = task.work.agent_query config = replace( load_config(), @@ -344,7 +348,7 @@ def main(argv=None): args.restore_goal, args.restore_agent, args.restore_source_todo, int(time.time()) + args.timeout, args.replan_after_turns or 6, tuple(args.restore_research_component), args.restore_fresh_replan, - destination_config=args.restore_model_config) + destination_config=args.restore_model_config, research_sidecar=research_sidecar) metadata = candidate.verify(task.cwd) if args.restore_model_config is not None: from loopx.hosts.codex_checkpoint import checkpoint_destination_config @@ -358,7 +362,7 @@ def main(argv=None): clock = attempt / "restored-phase-deadline" warm_start = candidate agent = SForgeWorker(config, profile=args.worker, cwd=task.cwd, - warm_start=warm_start, + warm_start=warm_start, research_sidecar=research_sidecar, timeout_seconds=args.timeout, feedback_prompt=feedback_prompt, feedback=args.feedback, workspace_instructions=workspace_instructions, task_objective=task_objective, @@ -403,6 +407,7 @@ def main(argv=None): oauth_proxy=bool(args.api_proxy_url or api_base_url), blind_api_endpoint=blind_endpoint, feedback=feedback, docker_host_bridge=args.docker_host_bridge, + research_sidecar=research_sidecar, **({"checkpoint": checkpoint} if checkpoint is not None else {})) for image in (task.work_image_key, task.judge_image_key): if not backend.image_exists(image): diff --git a/benchmark/runtime/research_artifacts.py b/benchmark/runtime/research_artifacts.py new file mode 100644 index 0000000000..76101190c7 --- /dev/null +++ b/benchmark/runtime/research_artifacts.py @@ -0,0 +1,66 @@ +"""Provider-owned research data outside the native submission tree. + +This composes the existing checkpoint component transport. It neither filters +submitted files nor changes the task's evaluator or feedback permissions. +""" +from __future__ import annotations + +from copy import deepcopy +from pathlib import PurePosixPath + + +RESEARCH_DIRECTORY = "/tmp/loopx-benchmark-research" +RESEARCH_COMPONENT = "benchmark_research" + + +def validate_research_scope(workspace: str, submit_paths: list[str]) -> None: + """Refuse an overlapping task instead of silently changing submission scope.""" + root = PurePosixPath(workspace) + research = PurePosixPath(RESEARCH_DIRECTORY) + roots = [root, *(PurePosixPath(path) if PurePosixPath(path).is_absolute() + else root / path for path in submit_paths)] + for path in roots: + if not path.is_absolute() or ".." in path.parts: + raise ValueError("Research storage requires normalized absolute task and submission paths") + if research.is_relative_to(path) or path.is_relative_to(research): + raise ValueError("Research storage overlaps the task or its submitted paths") + + +def with_research_capture(request: dict) -> dict: + """Add this runner's data to a copy of its explicit capture declaration.""" + result = deepcopy(request) + expected = {"id": RESEARCH_COMPONENT, "source_path": RESEARCH_DIRECTORY} + present = False + for component in result["components"]: + if component.get("id") == RESEARCH_COMPONENT: + if component != expected or present: + raise ValueError("Reserved research component conflicts with the checkpoint declaration") + present = True + else: + source = PurePosixPath(component["source_path"]) + research = PurePosixPath(RESEARCH_DIRECTORY) + if source.is_relative_to(research) or research.is_relative_to(source): + raise ValueError("Checkpoint component overlaps the reserved research directory") + if not present: + result["components"].append(expected) + return result + + +def selected_research_components(metadata: dict, selected, *, managed: bool): + """Only the runner's exact sealed component joins explicit receiver choices. + + Older checkpoints do not have this component. Their original workspace and + explicit research selections remain intact; no historical files are moved. + """ + if not managed: + return selected + if (not isinstance(selected, (tuple, list)) or + any(not isinstance(key, str) for key in selected) or len(set(selected)) != len(selected)): + raise ValueError("Research component selection must contain distinct captured IDs") + matches = [entry for entry in metadata["worker_file_snapshot"]["components"] + if entry["id"] == RESEARCH_COMPONENT] + if not matches: + return selected + if len(matches) != 1 or matches[0] != {"id": RESEARCH_COMPONENT, "source_path": RESEARCH_DIRECTORY}: + raise ValueError("Captured runner research component has a conflicting identity") + return tuple(selected) if RESEARCH_COMPONENT in selected else (*selected, RESEARCH_COMPONENT) diff --git a/benchmark/runtime/sforge.py b/benchmark/runtime/sforge.py index cdbf3048bc..de32fdc113 100644 --- a/benchmark/runtime/sforge.py +++ b/benchmark/runtime/sforge.py @@ -101,7 +101,7 @@ def __init__(self, config, *, profile: str, cwd: str, planning_horizon_goal: bool | None = None, checkpoint_request_path: str | None = None, checkpoint_request: bytes | None = None, - warm_start=None): + warm_start=None, research_sidecar: bool = False): super().__init__(config) if profile not in PROFILES: raise ValueError("Unknown benchmark worker profile") @@ -184,8 +184,13 @@ def __init__(self, config, *, profile: str, cwd: str, raise ValueError("checkpoint_request_path requires a heartbeat profile") from .checkpoint import normalize_checkpoint_request_path self.checkpoint_request_path = normalize_checkpoint_request_path(checkpoint_request_path) + if type(research_sidecar) is not bool: + raise ValueError("research_sidecar requires an explicit boolean") + self.research_sidecar = research_sidecar self.warm_start = warm_start if warm_start is not None: + if research_sidecar and not warm_start.research_sidecar: + raise ValueError("Managed research storage requires matching warm receiver selection") if not profile.startswith("heartbeat-") or feedback != "native" or self.task_entry != "loopx-planned": raise ValueError("Warm checkpoint continuation requires a planned native heartbeat worker") warm_start.verify(cwd) @@ -242,9 +247,26 @@ def install_stop_hook(self, backend, handle, log_dir, logger): # Solver execution must never inherit this setup-only proxy. self.environment.setup_env = {} + def _install_research_directory(self, backend, handle, *, restored: bool): + from .research_artifacts import RESEARCH_DIRECTORY + directory = shlex.quote(RESEARCH_DIRECTORY) + # Fresh images cannot contribute unrelated data at this reserved path. + # Warm restore has already verified bytes and modes; do not chmod them. + prepare = (f"test -d {directory} && test ! -L {directory}" if restored else + f"mkdir -m 0700 {directory}") + result = backend.exec_run(handle, ["/bin/sh", "-c", + f'test "$(readlink -f /tmp)" = /tmp && ' + f'workspace=$(readlink -f -- {shlex.quote(self.cwd)}) && test -n "$workspace" && ' + f'case {directory}/ in "$workspace/"*) exit 1;; esac && ' + f'case "$workspace/" in {directory}/*) exit 1;; esac && ' + f'{prepare} && chown agent:agent {directory}'], user="root") + if result.exit_code: + raise RuntimeError("Research directory is unavailable or conflicts with existing task data") + def _install_worker(self, backend, handle, log_dir, logger): self.backend, self.handle = backend, handle self.log_dir = log_dir + research_paths = [] effort = self._config.agent_effort common = dict(logs_dir=log_dir / "worker", model_name=self._config.agent_model, reasoning_effort=effort) @@ -362,6 +384,9 @@ def _install_worker(self, backend, handle, log_dir, logger): f"chmod 0600 {shlex.quote(self.checkpoint_request_path)}"], user="root") if result.exit_code: raise RuntimeError("Could not protect the explicit checkpoint declaration") + if self.research_sidecar: + from .research_artifacts import RESEARCH_DIRECTORY + self._install_research_directory(backend, handle, restored=RESEARCH_DIRECTORY in research_paths) # Prompt replacement describes the policy; the feedback mode owns access. if self.feedback != "native": result = backend.exec_run(handle, ["rm", "-f", "/usr/local/bin/sforge-submit"], user="root") @@ -397,6 +422,7 @@ def _install_worker(self, backend, handle, log_dir, logger): "task_entry": self.task_entry, "reasoning_effort": effort, "timeout_seconds": self.timeout_seconds, "loopx_usage_ping_enabled": False, + **({"research_directory": RESEARCH_DIRECTORY} if self.research_sidecar else {}), **({"checkpoint_capture_enabled": True} if self.checkpoint_request_path is not None else {}), **({"planning_timeout_seconds": None} @@ -416,8 +442,11 @@ def _install_worker(self, backend, handle, log_dir, logger): }, indent=2)) def format_run_cmd(self, prompt_path, *, model=None, cwd="", internet=True, resume=False): - if self.workspace_instructions is not None and not self.instructions_installed and self.warm_start is None: - self._install_workspace_instructions() + if self.workspace_instructions is not None and not self.instructions_installed: + if self.warm_start is None: + self._install_workspace_instructions() + elif self.research_sidecar: + self._install_restored_research_instructions() if self.feedback_prompt is not None and not self.prompt_installed: with tempfile.TemporaryDirectory(prefix="benchmark-prompt-") as directory: prompt = Path(directory) / "task.md" @@ -534,9 +563,21 @@ def _warm_command(self, action, *, user="agent"): def _install_workspace_instructions(self): """Preserve task instructions and verify the policy before model entry.""" + self._append_workspace_policy("", self.workspace_instructions) + + def _install_restored_research_instructions(self): + from benchmark.edgebench.prompts import research_artifact_instructions + # Preserve the sealed task instructions and earlier feedback policy. + # Only the runner-owned storage location changes in this continuation; + # the original checkpoint remains untouched and independently verifiable. + self._append_workspace_policy("", + "For this continuation, the following research storage instructions " + "replace earlier workspace-storage guidance. Existing implementation and " + "research artifacts are retained.\n\n" + research_artifact_instructions(checkpoint=True)) + + def _append_workspace_policy(self, marker: str, policy: str): path = shlex.quote(str(PurePosixPath(self.cwd) / "AGENTS.md")) - marker = "" - block = f"\n\n{marker}\n\n{self.workspace_instructions}" + block = f"\n\n{marker}\n\n{policy}" check = f"test ! -L {path} && {{ test ! -e {path} || test -f {path}; }}" read = asyncio.run(self.environment.exec(f"{check} && {{ test ! -f {path} || cat {path}; }}")) if read.return_code: diff --git a/benchmark/runtime/sforge_backend.py b/benchmark/runtime/sforge_backend.py index 78c13e4f73..0edc0652f7 100644 --- a/benchmark/runtime/sforge_backend.py +++ b/benchmark/runtime/sforge_backend.py @@ -19,9 +19,12 @@ class RecordingDockerBackend(DockerBackend): def __init__(self, *, log_dir: Path, logger, oauth_proxy: bool = False, blind_api_endpoint: tuple[str, int] | None = None, feedback=None, checkpoint=None, client=None, - docker_host_bridge: bool = False): + docker_host_bridge: bool = False, research_sidecar: bool = False): if not isinstance(docker_host_bridge, bool): raise ValueError("docker_host_bridge must be an explicit boolean") + if type(research_sidecar) is not bool: + raise ValueError("research_sidecar requires an explicit boolean") + self.research_sidecar = research_sidecar super().__init__(client) self.docker_host_bridge = docker_host_bridge self.log_dir, self.logger = log_dir, logger @@ -147,11 +150,15 @@ def cleanup_container(self, handle, logger=None): observations = [] # Never copy the whole Codex home: auth.json must stay private in # the source home and disposable trial, outside collected artifacts. - for remote, name in [("/logs/agent", "runtime"), + sources = [("/logs/agent", "runtime"), ("/home/agent/.codex/sessions", "native-sessions"), (f"{_CODEX_HOME}/sessions", "worker-sessions"), (_CONTROL, "worker-control"), - (_LOOPX_RUNTIME, "loopx-state")]: + (_LOOPX_RUNTIME, "loopx-state")] + if self.research_sidecar: + from .research_artifacts import RESEARCH_DIRECTORY + sources.append((RESEARCH_DIRECTORY, "research")) + for remote, name in sources: try: data = self.copy_from_container(handle, PurePosixPath(remote)) target = self.log_dir / name diff --git a/benchmark/runtime/sforge_warm_start.py b/benchmark/runtime/sforge_warm_start.py index f7f48fe0ab..516d54f80d 100644 --- a/benchmark/runtime/sforge_warm_start.py +++ b/benchmark/runtime/sforge_warm_start.py @@ -38,6 +38,11 @@ class WarmStart: research_component_ids: tuple[str, ...] = () fresh_replan: bool = False destination_config: Path | None = None + research_sidecar: bool = False + + def _selected_research(self, metadata: dict): + from .research_artifacts import selected_research_components + return selected_research_components(metadata, self.research_component_ids, managed=self.research_sidecar) def verify(self, workspace: str) -> dict: from loopx.hosts.checkpoint_retention import checkpoint_bank_use @@ -45,6 +50,8 @@ def verify(self, workspace: str) -> dict: return self._verify(workspace) def _verify(self, workspace: str) -> dict: + if type(self.research_sidecar) is not bool: + raise ValueError("research_sidecar requires an explicit boolean") if type(self.fresh_replan) is not bool: raise ValueError("fresh_replan requires an explicit boolean context treatment") if (type(self.normal_cadence) is not int or not 1 <= self.normal_cadence <= 6 @@ -76,7 +83,7 @@ def _verify(self, workspace: str) -> dict: from loopx.hosts.codex_checkpoint import checkpoint_destination_config checkpoint_destination_config(self.config, native["config_sha256"], self.destination_config) _workspace_component(self.root, metadata, Path(workspace)) - _research_components(self.root, metadata, Path(workspace), self.research_component_ids) + _research_components(self.root, metadata, Path(workspace), self._selected_research(metadata)) context = metadata["research_context"] context_path = self.root / "components" / context["component_id"] / context["member_path"] if not context_path.resolve(strict=True).is_relative_to(self.root / "components"): @@ -102,7 +109,7 @@ async def _stage(self, environment, workspace: str, *, effective_config: bytes | None = None, provider_base_url: str | None = None) -> list[str]: metadata = self._verify(workspace) - research = _research_components(self.root, metadata, Path(workspace), self.research_component_ids) + research = _research_components(self.root, metadata, Path(workspace), self._selected_research(metadata)) requested = self.destination_config or self.config transport_config = None if effective_config is not None: @@ -150,8 +157,8 @@ async def _stage(self, environment, workspace: str, *, await environment.upload_file(self.destination_config, target) payload["destination_config"] = target protected_files.append(target) - if self.research_component_ids: - payload["research_component_ids"] = list(self.research_component_ids) + if research: + payload["research_component_ids"] = [key for key, _, _ in research] if self.fresh_replan: payload["fresh_replan"] = True with tempfile.TemporaryDirectory() as folder: diff --git a/benchmark/tests/test_edgebench_feedback.py b/benchmark/tests/test_edgebench_feedback.py index 073ca1fb2c..b48837c530 100644 --- a/benchmark/tests/test_edgebench_feedback.py +++ b/benchmark/tests/test_edgebench_feedback.py @@ -367,6 +367,8 @@ def backend(**kwargs): def handoff(**kwargs): selected = mode or "native" assert kwargs["agent"].feedback == selected + assert kwargs["agent"].research_sidecar is (selected == "native") + assert constructed["research_sidecar"] is (selected == "native") assert kwargs["max_submissions"] == (None if selected == "native" else 0) assert (constructed["feedback"] is not None) == (selected == "best-only") assert ("host_registration_callback" in kwargs) == (selected == "best-only") diff --git a/benchmark/tests/test_edgebench_feedback_docker.py b/benchmark/tests/test_edgebench_feedback_docker.py index 098bfec053..4c8ce15d70 100644 --- a/benchmark/tests/test_edgebench_feedback_docker.py +++ b/benchmark/tests/test_edgebench_feedback_docker.py @@ -93,6 +93,42 @@ def test_native_execution_disposition_uses_real_docker_exit(tmp_path, agent_imag backend.client.close() +def test_warm_research_policy_preserves_old_task_and_refuses_conflicting_override( + tmp_path, agent_image, monkeypatch): + from sforge.harness.config import SForgeConfig + from benchmark.runtime.sforge import SForgeWorker, SForgeEnvironment + from benchmark.runtime.sforge_backend import RecordingDockerBackend + from benchmark.runtime.research_artifacts import RESEARCH_DIRECTORY + monkeypatch.setenv("CODEX_AUTH_JSON_PATH", "/synthetic-unused-credential") + worker = SForgeWorker(SForgeConfig(agent_model="fixture", agent_effort="xhigh"), + profile="heartbeat-explore", cwd="/task", research_sidecar=True) + backend = RecordingDockerBackend(log_dir=tmp_path / "collected", + logger=logging.getLogger("research-migration-smoke"), oauth_proxy=True) + handle = backend.create_container(agent_image.id, "research-migration-" + uuid.uuid4().hex[:10], + cpu_limit=1, mem_limit="256m") + backend.start_container(handle) + prior = "Original task acceptance.\n\n\n\nKeep notes in the workspace.\n" + try: + command = f"mkdir /task && printf %s {shlex.quote(prior)} > /task/AGENTS.md && chown -R agent:agent /task" + assert backend.exec_run(handle, ["sh", "-c", command], user="root").exit_code == 0 + worker.environment = SForgeEnvironment(backend, handle) + worker.log_dir = tmp_path + worker._install_restored_research_instructions() + def read(): + return backend.exec_run(handle, ["cat", "/task/AGENTS.md"], user="agent").output + after = read() + assert after.startswith(prior) and RESEARCH_DIRECTORY in after + assert "replace earlier workspace-storage guidance" in after + worker._install_restored_research_instructions() + assert read() == after + with pytest.raises(RuntimeError, match="conflicting or partial"): + worker._append_workspace_policy("", "foreign override") + assert read() == after + finally: + backend.cleanup_container(handle) + backend.client.close() + + def test_host_bridge_pin_reaches_only_the_native_allowed_port(tmp_path, agent_image): from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from sforge.harness.network_isolation import build_allowed_endpoints @@ -884,13 +920,13 @@ def test_checkpoint_storage_policy_is_installed_in_real_worker(tmp_path, agent_i from sforge.harness.config import SForgeConfig from benchmark.runtime.sforge import SForgeWorker, SForgeEnvironment from benchmark.runtime.sforge_backend import RecordingDockerBackend - from benchmark.edgebench.prompts import native_task_instructions, checkpoint_research_instructions + from benchmark.edgebench.prompts import native_task_instructions, research_artifact_instructions monkeypatch.setenv("CODEX_AUTH_JSON_PATH", "/synthetic-unused-credential") request = {"destination": "/logs/agent/checkpoints", "point_metadata": {}, "components": [{"id": "workspace", "source_path": "/task"}]} policy = native_task_instructions(["solver.py"], submission_cooldown=3600, eval_interval=0, internet=False, selection="score_first", score_direction="maximize") - policy += checkpoint_research_instructions(request, "/task") + policy += research_artifact_instructions(checkpoint=True) worker = SForgeWorker(SForgeConfig(agent_model="fixture", agent_effort="xhigh"), profile="heartbeat-explore", cwd="/task", checkpoint_request=json.dumps(request).encode(), workspace_instructions=policy, feedback_prompt=policy + "---\n\nOriginal acceptance.\n") @@ -926,3 +962,47 @@ def test_checkpoint_storage_policy_is_installed_in_real_worker(tmp_path, agent_i finally: backend.cleanup_container(handle) backend.client.close() + + +@pytest.mark.parametrize("profile", ["official", "native-goal", "heartbeat-resume", "heartbeat-explore"]) +def test_external_research_survives_collection_without_entering_native_submission( + tmp_path, agent_image, monkeypatch, profile): + from sforge.harness.config import SForgeConfig + from sforge.harness.run_agent import _extract_archive_from_container + from benchmark.runtime.sforge import SForgeWorker + from benchmark.runtime.sforge_backend import RecordingDockerBackend + from benchmark.runtime.research_artifacts import RESEARCH_DIRECTORY + monkeypatch.setenv("CODEX_AUTH_JSON_PATH", "/synthetic-unused-credential") + worker = SForgeWorker(SForgeConfig(agent_model="fixture", agent_effort="xhigh"), + profile=profile, cwd="/task", research_sidecar=True) + backend = RecordingDockerBackend(log_dir=tmp_path / "collected", research_sidecar=True, + logger=logging.getLogger("research-storage-smoke"), oauth_proxy=True) + handle = backend.create_container(agent_image.id, "research-storage-" + uuid.uuid4().hex[:10], + cpu_limit=1, mem_limit="256m") + backend.start_container(handle) + try: + assert backend.exec_run(handle, ["sh", "-c", "mkdir /task && chown agent:agent /task"], + user="root").exit_code == 0 + worker._install_research_directory(backend, handle, restored=False) + with pytest.raises(RuntimeError, match="conflicts"): + worker._install_research_directory(backend, handle, restored=False) + # Synthetic content is deliberately identical: packaging is location- + # based, never a hidden-word filter that could remove real candidate code. + payload = "candidate sentinel: FORBIDDEN_EXTERNAL_CALL" + for remote in ("/task/candidate.txt", RESEARCH_DIRECTORY + "/observations.json"): + command = f"printf %s {shlex.quote(payload)} > {shlex.quote(remote)}" + assert backend.exec_run(handle, ["sh", "-c", command], user="agent").exit_code == 0 + task = SimpleNamespace(cwd="/task", submit_paths=["."], submit_exclude=[]) + archive = _extract_archive_from_container(backend, handle, task) + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as bundle: + names = bundle.getnames() + assert "./candidate.txt" in names + assert bundle.extractfile("./candidate.txt").read().decode() == payload + assert all("observations" not in name and "research" not in name for name in names) + finally: + backend.cleanup_container(handle) + backend.client.close() + collected = list((tmp_path / "collected/research").rglob("observations.json")) + assert len(collected) == 1 and collected[0].read_text() == payload + receipt = json.loads((tmp_path / "collected/artifact-collection.json").read_bytes()) + assert {"artifact": "research", "collected": True} in receipt diff --git a/benchmark/tests/test_edgebench_prompts.py b/benchmark/tests/test_edgebench_prompts.py index 3e30ad752b..ad753c6d0b 100644 --- a/benchmark/tests/test_edgebench_prompts.py +++ b/benchmark/tests/test_edgebench_prompts.py @@ -472,18 +472,16 @@ async def prepare(environment, text, **kwargs): assert remote.read_text() == prompt -@pytest.mark.parametrize("source,covered", [ - ("/task", True), ("/home/workspace", True), - ("/home/workspace/task/solver.py", False), ("/home/workspace/task-other", False), -]) -def test_checkpoint_guidance_distinguishes_directory_from_file_coverage(source, covered): - from benchmark.edgebench.prompts import checkpoint_research_instructions - workspace = "/task" if source == "/task" else "/home/workspace/task" - text = checkpoint_research_instructions( - {"components": [{"id": "code", "source_path": source}]}, workspace) - assert ("includes this task workspace" in text) is covered - assert ("does not cover the whole task workspace" in text) is not covered - assert "negative results" in text and "workspace-relative references" in text +@pytest.mark.parametrize("checkpoint", [False, True]) +def test_research_guidance_separates_candidate_inputs_and_capture(checkpoint): + from benchmark.edgebench.prompts import research_artifact_instructions + from benchmark.runtime.research_artifacts import RESEARCH_DIRECTORY + text = research_artifact_instructions(checkpoint=checkpoint) + assert RESEARCH_DIRECTORY in text + assert "outside the task workspace and submitted paths" in text + assert "negative results" in text and "runtime/build data" in text assert "external file does not preserve its bytes" in text assert "does not change submitted files" in text - assert source not in text # Do not echo arbitrary private capture roots. + assert ("verified continuation restores" in text) is checkpoint + assert ("does not provide a resumable checkpoint" in text) is not checkpoint + assert "does not relocate or discard historical evidence" in text diff --git a/benchmark/tests/test_research_artifacts.py b/benchmark/tests/test_research_artifacts.py new file mode 100644 index 0000000000..0c819e560f --- /dev/null +++ b/benchmark/tests/test_research_artifacts.py @@ -0,0 +1,117 @@ +"""Research retention must not silently alter a native candidate submission.""" +from copy import deepcopy + +import pytest + +from benchmark.runtime.research_artifacts import ( + RESEARCH_COMPONENT, RESEARCH_DIRECTORY, selected_research_components, + validate_research_scope, with_research_capture, +) + + +def test_whole_workspace_submission_and_explicit_files_allow_external_research(): + validate_research_scope("/task", ["."]) + validate_research_scope("/task", ["solver.py", "/opt/candidate"]) + + +@pytest.mark.parametrize("workspace, paths", [ + ("/tmp", ["."]), ("/task", [RESEARCH_DIRECTORY]), + (RESEARCH_DIRECTORY + "/task", ["."]), ("/task", ["/tmp"]), + ("relative", ["."]), ("/task", ["../tmp"]), +]) +def test_ambiguous_or_overlapping_submission_scope_is_not_rewritten(workspace, paths): + before = deepcopy(paths) + with pytest.raises(ValueError): + validate_research_scope(workspace, paths) + assert paths == before + + +def test_capture_adds_only_reserved_component_and_never_changes_original_request(): + request = {"destination": "/logs/checkpoints", "point_metadata": {"fixture": True}, + "components": [{"id": "workspace", "source_path": "/task"}]} + before = deepcopy(request) + captured = with_research_capture(request) + assert request == before + assert captured["components"] == [*before["components"], + {"id": RESEARCH_COMPONENT, "source_path": RESEARCH_DIRECTORY}] + assert with_research_capture(captured) == captured + assert captured["point_metadata"] == before["point_metadata"] + + +@pytest.mark.parametrize("entry", [ + {"id": RESEARCH_COMPONENT, "source_path": "/tmp/unrelated"}, + {"id": "unrelated", "source_path": RESEARCH_DIRECTORY}, + {"id": "parent", "source_path": "/tmp"}, + {"id": "child", "source_path": RESEARCH_DIRECTORY + "/notes"}, +]) +def test_capture_conflicts_preserve_original_declaration(entry): + request = {"components": [entry]} + before = deepcopy(request) + with pytest.raises(ValueError): + with_research_capture(request) + assert request == before + + +def test_only_exact_managed_component_is_selected_without_an_operator_override(): + entries = [{"id": "other-research", "source_path": "/tmp/other"}] + metadata = {"worker_file_snapshot": {"components": entries}} + assert selected_research_components(metadata, (), managed=True) == () + entries.append({"id": RESEARCH_COMPONENT, "source_path": RESEARCH_DIRECTORY}) + assert selected_research_components(metadata, (), managed=True) == (RESEARCH_COMPONENT,) + assert selected_research_components(metadata, (), managed=False) == () + assert selected_research_components(metadata, ("other-research",), managed=True) == ( + "other-research", RESEARCH_COMPONENT) + entries[-1]["source_path"] = "/tmp/foreign" + with pytest.raises(ValueError, match="conflicting identity"): + selected_research_components(metadata, (), managed=True) + + +@pytest.mark.parametrize("profile", ["official", "native-goal", "heartbeat-resume", "heartbeat-explore"]) +def test_native_entry_installs_identical_research_scope_and_declares_checkpoint_coverage( + tmp_path, monkeypatch, profile): + pytest.importorskip("sforge") + pytest.importorskip("harbor") + import json + from types import SimpleNamespace + from benchmark.edgebench import run + monkeypatch.setenv("LOOPX_SRC_DIR", str(tmp_path)) + monkeypatch.setenv("LOOPX_EXPECTED_COMMIT", "fixture") + monkeypatch.setenv("CODEX_AUTH_JSON_PATH", "/synthetic-login") + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + (tmp_path / "fixture.json").write_text("{}") + monkeypatch.setattr(run, "source_pins", lambda *a: ("fixture", "fixture")) + monkeypatch.setattr(run, "load_benchmark", lambda *a: None) + monkeypatch.setattr(run, "make_task_spec", lambda *a: SimpleNamespace( + cwd="/task", work_image_key="work", judge_image_key="judge", internet=False, + game_mode=False, submit_paths=["."], work=SimpleNamespace(agent_query="Original acceptance."), + judge=SimpleNamespace(selection="score_first", score_direction="maximize"))) + def backend(**kwargs): + assert kwargs["research_sidecar"] is True + return SimpleNamespace(image_exists=lambda image: True) + monkeypatch.setattr(run, "RecordingDockerBackend", backend) + capture = profile.startswith("heartbeat-") + request = {"destination": "/logs/agent/checkpoints", "components": [ + {"id": "workspace", "source_path": "/task"}], "point_metadata": {}} + source = tmp_path / "capture.json" + source.write_text(json.dumps(request)) + def boundary(**kwargs): + worker = kwargs["agent"] + assert worker.research_sidecar is True + assert RESEARCH_DIRECTORY in worker.workspace_instructions + assert worker.feedback_prompt.startswith(worker.workspace_instructions) + assert worker.task_objective == "Original acceptance." + if capture: + assert json.loads(worker.checkpoint_request) == with_research_capture(request) + else: + assert worker.checkpoint_request is None + raise RuntimeError("synthetic launch boundary") + monkeypatch.setattr(run, "run_agent", boundary) + argv = ["--task", "fixture", "--tasks-dir", str(tmp_path), "--log-dir", str(tmp_path), + "--run-id", "fixture", "--worker", profile, "--model", "fixture", "--effort", "xhigh", + "--judge-url", "http://127.0.0.1:9999", "--feedback", "native"] + if capture: + argv += ["--checkpoint-request", str(source)] + with pytest.raises(RuntimeError, match="synthetic launch boundary"): + run.main(argv) + assert json.loads(source.read_bytes()) == request diff --git a/benchmark/tests/test_sforge_runtime.py b/benchmark/tests/test_sforge_runtime.py index e35ec3374c..a51417c7ed 100644 --- a/benchmark/tests/test_sforge_runtime.py +++ b/benchmark/tests/test_sforge_runtime.py @@ -430,11 +430,13 @@ async def installed(self, environment): "/python", "-m", "benchmark.runtime.worker"] -def test_artifact_collection_preserves_both_session_homes_without_auth(tmp_path, monkeypatch): +@pytest.mark.parametrize("research_sidecar", [False, True]) +def test_artifact_collection_preserves_both_session_homes_without_auth(tmp_path, monkeypatch, research_sidecar): pytest.importorskip("sforge") pytest.importorskip("harbor") from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend backend = object.__new__(RecordingDockerBackend) + backend.research_sidecar = research_sidecar backend.checkpoint = None backend.log_dir = tmp_path / "artifacts" backend.feedback = None @@ -459,6 +461,7 @@ def archive(handle, remote): assert "/home/agent/.codex" not in copied assert "/opt/loopx-benchmark/codex-home" not in copied assert all("auth.json" not in path for path in copied) + assert ("/tmp/loopx-benchmark-research" in copied) is research_sidecar assert cleaned == ["fixture-container"] assert all(row["collected"] for row in json.loads( (backend.log_dir / "artifact-collection.json").read_text())) @@ -475,6 +478,7 @@ def test_provider_observation_failure_cannot_skip_native_cleanup(tmp_path, monke pytest.importorskip("harbor") from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend backend = object.__new__(RecordingDockerBackend) + backend.research_sidecar = False backend.checkpoint = None backend.log_dir = tmp_path / "artifacts" backend.feedback = None @@ -526,6 +530,7 @@ def test_blind_policy_removes_judge_route_and_credentials_native_is_unchanged(mo pytest.importorskip("harbor") from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend, AllowedEndpoint backend = object.__new__(RecordingDockerBackend) + backend.research_sidecar = False backend.checkpoint = None backend.auth_ips = [] backend.blind_api_endpoint = None @@ -1088,6 +1093,7 @@ def test_recorded_solver_exit_does_not_take_status_from_output(tmp_path, monkeyp from types import SimpleNamespace from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend backend = object.__new__(RecordingDockerBackend) + backend.research_sidecar = False backend.checkpoint = None backend.log_dir = tmp_path / "collected" backend.blind_api_endpoint = None @@ -1420,7 +1426,7 @@ def test_native_checkpoint_storage_guidance_reaches_both_prompt_surfaces(tmp_pat pytest.importorskip("harbor") from types import SimpleNamespace from benchmark.edgebench import run - from benchmark.edgebench.prompts import native_task_prompt, native_task_instructions + from benchmark.edgebench.prompts import research_artifact_instructions, native_task_instructions monkeypatch.setenv("LOOPX_SRC_DIR", str(tmp_path)) monkeypatch.setenv("LOOPX_EXPECTED_COMMIT", "fixture") @@ -1440,12 +1446,12 @@ def before_solver(**kwargs): assert worker.task_objective == "Original acceptance." assert worker.feedback_prompt.startswith(worker.workspace_instructions) assert worker.feedback_prompt.endswith("---\n\nOriginal acceptance.\n") - assert ("### Durable Research Artifacts" in worker.workspace_instructions) is capture - if not capture: - policy = dict(submission_cooldown=3600, eval_interval=0, internet=False, - selection="score_first", score_direction="maximize") - assert worker.workspace_instructions == native_task_instructions(["solver.py"], **policy) - assert worker.feedback_prompt == native_task_prompt("Original acceptance.", ["solver.py"], **policy) + assert worker.research_sidecar is True + assert ("verified continuation restores" in worker.workspace_instructions) is capture + policy = dict(submission_cooldown=3600, eval_interval=0, internet=False, + selection="score_first", score_direction="maximize") + assert worker.workspace_instructions == (native_task_instructions(["solver.py"], **policy) + + research_artifact_instructions(checkpoint=capture)) raise RuntimeError("validated before model execution") monkeypatch.setattr(run, "run_agent", before_solver) args = ["--task", "fixture", "--tasks-dir", str(tmp_path), "--log-dir", str(tmp_path), diff --git a/benchmark/tests/test_sforge_warm_start.py b/benchmark/tests/test_sforge_warm_start.py index e000dcc043..11d659c87b 100644 --- a/benchmark/tests/test_sforge_warm_start.py +++ b/benchmark/tests/test_sforge_warm_start.py @@ -120,7 +120,7 @@ def point(tmp_path, request, monkeypatch): @pytest.mark.parametrize("change", [ {"goal_id": "foreign-goal"}, {"agent_id": "foreign-agent"}, {"source_todo_id": "missing"}, {"deadline_epoch": 1}, {"deadline_epoch": True}, {"normal_cadence": 0}, {"normal_cadence": True}, - {"fresh_replan": "false"}, + {"fresh_replan": "false"}, {"research_sidecar": "false"}, ]) def test_invalid_participant_rejects_before_workspace_or_authority_changes(point, change): start, workspace = point @@ -385,6 +385,8 @@ def at_source_check(*args): def research_point(point, tmp_path, monkeypatch, request): start, workspace = point research = tmp_path / "retained-experiments" + managed = getattr(request, "param", None) == "managed" + component_id = "benchmark_research" if managed else "experiments" if getattr(request, "param", "directory") == "file": research.write_text("retained observations") research.chmod(0o600) @@ -395,7 +397,7 @@ def research_point(point, tmp_path, monkeypatch, request): (research / "observations.json").write_text('[2, 3, 5]') (research / "candidate.py").chmod(0o600) manifest = json.loads((start.root / "checkpoint.json").read_bytes()) - components = manifest["components"] + [{"id": "experiments", "source_path": str(research)}] + components = manifest["components"] + [{"id": component_id, "source_path": str(research)}] metadata = manifest["point_metadata"] metadata["worker_file_snapshot"]["components"].append(components[-1]) probe = effect_runtime_result("runtime.checkpoint_artifacts", {"action": "probe", "components": components}) @@ -408,7 +410,11 @@ def research_point(point, tmp_path, monkeypatch, request): else: research.unlink() monkeypatch.setattr(warm, "_RESEARCH_ROOTS", (tmp_path,)) - return replace(start, root=unit, sha256=result["sha256"], research_component_ids=("experiments",)), workspace, research + if managed: + from benchmark.runtime import research_artifacts + monkeypatch.setattr(research_artifacts, "RESEARCH_DIRECTORY", str(research)) + return replace(start, root=unit, sha256=result["sha256"], research_sidecar=managed, + research_component_ids=() if managed else ("experiments",)), workspace, research def test_explicit_research_files_reproduce_the_retained_experiment(research_point): @@ -434,6 +440,47 @@ def test_unselected_research_files_remain_inert(research_point): assert not research.exists() +@pytest.mark.parametrize("research_point", ["managed"], indirect=True) +def test_managed_research_roundtrips_through_existing_capture_and_receiver(research_point, monkeypatch, tmp_path): + start, workspace, research = research_point + original_manifest = (start.root / "checkpoint.json").read_bytes() + control = tmp_path / "control" + control.mkdir() + monkeypatch.setattr(warm, "_CONTROL", str(control)) + class Environment: + async def upload_dir(self, source, target): + shutil.copytree(source, target) + async def upload_file(self, source, target): + shutil.copyfile(source, target) + async def exec(self, command, **kwargs): + return SimpleNamespace(return_code=0) + assert start.research_component_ids == () # No extra manual recovery flag. + assert asyncio.run(start.stage(Environment(), str(workspace)))[0] == str(research) + declaration = json.loads((control / "warm-start.json").read_bytes()) + assert declaration["research_component_ids"] == ["benchmark_research"] + warm.restore_workspace(declaration, workspace) + scope = {} + exec((research / "candidate.py").read_text(), scope) + assert [scope["predict"](x) for x in json.loads((research / "observations.json").read_text())] == [14, 21, 35] + assert (research / "candidate.py").stat().st_mode & 0o777 == 0o600 + assert (workspace / "candidate").read_text() == "retained candidate" + assert (start.root / "checkpoint.json").read_bytes() == original_manifest + with pytest.raises(ValueError, match="already exists"): + warm.restore_workspace(declaration, workspace) + + +def test_legacy_checkpoint_keeps_original_workspace_and_does_not_invent_research(point): + start, workspace = point + original_manifest = (start.root / "checkpoint.json").read_bytes() + start = replace(start, research_sidecar=True) + metadata = start.verify(str(workspace)) + assert start._selected_research(metadata) == () + warm.restore_workspace({"root": str(start.root), "sha256": start.sha256, + "deadline_epoch": start.deadline_epoch}, workspace) + assert (workspace / "candidate").read_text() == "retained candidate" + assert (start.root / "checkpoint.json").read_bytes() == original_manifest + + @pytest.mark.parametrize("selected", [("absent",), ("workspace",), ("context",), ("experiments", "experiments"), (None,), "experiments"]) def test_invalid_research_selection_rejects_before_workspace_changes(research_point, selected):