diff --git a/clearwing/sandbox/hunter_sandbox.py b/clearwing/sandbox/hunter_sandbox.py index c0e87f2f..a0d6fbd0 100644 --- a/clearwing/sandbox/hunter_sandbox.py +++ b/clearwing/sandbox/hunter_sandbox.py @@ -79,8 +79,12 @@ def __init__( default_cpus: float | None = None, backend: SandboxBackend | None = None, gvisor_runtime: str | None = None, + default_memory_mb: int = 4096, ): self._validate_cpu_limit(default_cpus, name="default_cpus") + if type(default_memory_mb) is not int or not 512 <= default_memory_mb <= 16384: + raise ValueError("default_memory_mb must be an integer between 512 and 16384") + self.default_memory_mb = default_memory_mb self.repo_path = os.path.abspath(repo_path) self.languages = languages or [] self.sanitizers = sanitizers or ["asan", "ubsan"] @@ -315,7 +319,7 @@ def _build_variant_image(self, sanitizers: list[str]) -> str: def spawn( self, session_id: str | None = None, - memory_mb: int = 2048, + memory_mb: int | None = None, timeout_seconds: int = 300, scratch_mount: bool = True, variant: list[str] | None = None, @@ -383,7 +387,7 @@ def spawn( policy="sourcehunt", isolation="enhanced" if runtime else "default", mounts=mounts, - memory_mb=memory_mb, + memory_mb=self.default_memory_mb if memory_mb is None else memory_mb, cpus=resolved_cpus, timeout_seconds=timeout_seconds, env=env, diff --git a/clearwing/sourcehunt/commit_monitor.py b/clearwing/sourcehunt/commit_monitor.py index 49b3acb8..48f85e69 100644 --- a/clearwing/sourcehunt/commit_monitor.py +++ b/clearwing/sourcehunt/commit_monitor.py @@ -45,6 +45,7 @@ class CommitMonitorConfig: depth: str = "standard" budget_usd: float = 0.0 sandbox_cpus: float | None = None + sandbox_memory_mb: int | None = None enable_semgrep: bool = False on_finding: Callable | None = None runner_factory: Callable | None = None # test injection point @@ -277,6 +278,7 @@ def _run_sourcehunt_on(self, files: list[str]) -> list[Finding]: output_dir=self.config.output_dir, provider_manager=provider_manager, sandbox_cpus=self.config.sandbox_cpus, + sandbox_memory_mb=self.config.sandbox_memory_mb, enable_semgrep=self.config.enable_semgrep, ) try: diff --git a/clearwing/sourcehunt/config.py b/clearwing/sourcehunt/config.py index df160341..705392f6 100644 --- a/clearwing/sourcehunt/config.py +++ b/clearwing/sourcehunt/config.py @@ -102,6 +102,7 @@ class HuntTuning: campaign_hint: str | None = None gvisor_runtime: str | None = None sandbox_cpus: float | None = None # None = auto, 0 = unlimited + sandbox_memory_mb: int | None = None # None = 4096 MiB nday_filter_batch_size: int = 10 reveng_batch_size: int = 8 # Per-field character cap for trace-step code_snippet and note strings. diff --git a/clearwing/sourcehunt/proof/engine.py b/clearwing/sourcehunt/proof/engine.py index ef7e04e7..a013ca29 100644 --- a/clearwing/sourcehunt/proof/engine.py +++ b/clearwing/sourcehunt/proof/engine.py @@ -88,6 +88,7 @@ class ProofRunConfig: falsify: bool = True gvisor_runtime: str | None = None sandbox_cpus: float | None = None + sandbox_memory_mb: int | None = None evaluation_hints: dict[str, Any] = field(default_factory=dict) @@ -1007,10 +1008,13 @@ def _build_sandbox_runner( default_cpus=self.config.sandbox_cpus, ) self._sandbox.prepare_environment() - container = self._sandbox.spawn( - session_id=self.config.session_id or None, - runtime=self.config.gvisor_runtime, - ) + spawn_kwargs = { + "session_id": self.config.session_id or None, + "runtime": self.config.gvisor_runtime, + } + if self.config.sandbox_memory_mb is not None: + spawn_kwargs["memory_mb"] = self.config.sandbox_memory_mb + container = self._sandbox.spawn(**spawn_kwargs) except Exception as exc: raise ProofPreflightError( f"Unable to start the required C/C++ analysis sandbox: {exc}", diff --git a/clearwing/sourcehunt/runner.py b/clearwing/sourcehunt/runner.py index a45361e3..3f4c3782 100644 --- a/clearwing/sourcehunt/runner.py +++ b/clearwing/sourcehunt/runner.py @@ -420,6 +420,7 @@ def __init__( respect_gitignore: bool = False, live: bool = False, sandbox_cpus: float | None = None, + sandbox_memory_mb: int | None = None, *, config: SourceHuntConfig | None = None, flow: str = "legacy", @@ -558,6 +559,9 @@ def __init__( campaign_hint = campaign_hint if campaign_hint is not None else h.campaign_hint gvisor_runtime = gvisor_runtime if gvisor_runtime is not None else h.gvisor_runtime sandbox_cpus = sandbox_cpus if sandbox_cpus is not None else h.sandbox_cpus + sandbox_memory_mb = ( + sandbox_memory_mb if sandbox_memory_mb is not None else h.sandbox_memory_mb + ) trace_step_max_chars = ( h.trace_step_max_chars if trace_step_max_chars is None else trace_step_max_chars ) @@ -629,6 +633,10 @@ def __init__( trace_step_max_chars = max(0, trace_step_max_chars) if sandbox_cpus is not None and (not math.isfinite(sandbox_cpus) or sandbox_cpus < 0): raise ValueError("sandbox_cpus must be a finite number greater than or equal to 0") + if sandbox_memory_mb is not None and ( + type(sandbox_memory_mb) is not int or not 512 <= sandbox_memory_mb <= 16384 + ): + raise ValueError("sandbox_memory_mb must be an integer between 512 and 16384") normalized_target_files = self._normalize_target_files(target_files or ()) target_window_lines = 480 if target_window_lines is None else target_window_lines if type(target_window_lines) is not int or not 40 <= target_window_lines <= 500: @@ -788,6 +796,7 @@ def __init__( self._enable_behavior_monitor = enable_behavior_monitor self._enable_artifact_store = enable_artifact_store self._sandbox_cpus = None if sandbox_cpus is None else float(sandbox_cpus) + self._sandbox_memory_mb = sandbox_memory_mb self._gvisor_runtime = self._check_runtime_available(gvisor_runtime) self._preprocessing = preprocessing self._enable_semgrep = enable_semgrep @@ -1308,6 +1317,7 @@ def model_client(route: str) -> AsyncLLMClient | None: falsify=self._falsify, gvisor_runtime=self._gvisor_runtime, sandbox_cpus=self._sandbox_cpus, + sandbox_memory_mb=self._sandbox_memory_mb, evaluation_hints=evaluation_hints, ), model_client_factory=model_client, @@ -4250,6 +4260,7 @@ def _ensure_sandbox_factory(self, repo_path: str, files: list[FileTarget]) -> No languages=languages, deep_agent_mode=use_deep, default_cpus=self._sandbox_cpus, + default_memory_mb=self._sandbox_memory_mb or 4096, gvisor_runtime=self._gvisor_runtime, ) environment_ref = manager.prepare_environment() @@ -4289,7 +4300,6 @@ def _ensure_sandbox_factory(self, repo_path: str, files: list[FileTarget]) -> No if use_deep: self.sandbox_factory = lambda **kw: manager.spawn( writable_workspace=True, - memory_mb=kw.pop("memory_mb", 16384), timeout_seconds=kw.pop("timeout_seconds", 30), runtime=kw.pop("runtime", gvisor_rt), **kw, diff --git a/clearwing/ui/commands/sourcehunt.py b/clearwing/ui/commands/sourcehunt.py index 478a720e..931a6def 100644 --- a/clearwing/ui/commands/sourcehunt.py +++ b/clearwing/ui/commands/sourcehunt.py @@ -421,6 +421,13 @@ def add_parser(subparsers): metavar="N", help="CPU limit per sandbox (default: auto; 0 disables the limit)", ) + parser.add_argument( + "--sandbox-memory-mb", + type=int, + default=None, + metavar="MiB", + help="Memory limit per sandbox in MiB (default: 4096)", + ) parser.add_argument( "--tier-split", default="70/25/5", @@ -1177,6 +1184,7 @@ def handle(cli, args): depth=args.depth, budget_usd=args.budget, sandbox_cpus=args.sandbox_cpus, + sandbox_memory_mb=args.sandbox_memory_mb, enable_semgrep=args.semgrep, output_dir=args.output_dir, enable_github_checks=args.github_checks, @@ -1223,6 +1231,7 @@ def handle(cli, args): depth=args.depth, budget_usd=args.budget, sandbox_cpus=args.sandbox_cpus, + sandbox_memory_mb=args.sandbox_memory_mb, enable_semgrep=args.semgrep, enable_github_checks=args.github_checks, github_check_name=args.github_check_name, @@ -1307,6 +1316,7 @@ def handle(cli, args): enable_semgrep=args.semgrep, live=args.live, sandbox_cpus=args.sandbox_cpus, + sandbox_memory_mb=args.sandbox_memory_mb, flow=args.flow, proof_compile_commands=args.compile_commands, proof_validation_manifest=args.validation_manifest, diff --git a/tests/test_sandbox_writable_workspace.py b/tests/test_sandbox_writable_workspace.py index a2ba9c6f..46cfe884 100644 --- a/tests/test_sandbox_writable_workspace.py +++ b/tests/test_sandbox_writable_workspace.py @@ -130,9 +130,10 @@ def test_spawn_inherits_manager_default(self, tmp_path): sandbox = manager.spawn(scratch_mount=False) assert sandbox.config.cpus == 1.0 + assert sandbox.config.memory_mb == 4096 def test_spawn_override_wins_over_manager_default(self, tmp_path): - manager = HunterSandbox(repo_path=str(tmp_path), default_cpus=1.0) + manager = HunterSandbox(repo_path=str(tmp_path), default_cpus=1.0, default_memory_mb=6144) with ( patch.object( @@ -140,9 +141,24 @@ def test_spawn_override_wins_over_manager_default(self, tmp_path): ), patch.object(SandboxContainer, "start", return_value="cid"), ): - sandbox = manager.spawn(scratch_mount=False, cpus=2.5) + sandbox = manager.spawn(scratch_mount=False, cpus=2.5, memory_mb=8192) assert sandbox.config.cpus == 2.5 + assert sandbox.config.memory_mb == 8192 + + with ( + patch.object( + HunterSandbox, "_prepare_variant_environment", return_value="sandbox:test" + ), + patch.object(SandboxContainer, "start", return_value="cid"), + ): + inherited = manager.spawn(scratch_mount=False) + assert inherited.config.memory_mb == 6144 + + @pytest.mark.parametrize("value", [0, 511, 16385, 1.5, True, "4096"]) + def test_invalid_manager_memory_limit_rejected(self, tmp_path, value): + with pytest.raises(ValueError, match="default_memory_mb"): + HunterSandbox(repo_path=str(tmp_path), default_memory_mb=value) class TestSourceHuntSandboxCpuWiring: @@ -168,13 +184,13 @@ def test_runner_passes_override_to_manager_and_removes_hardcoded_limit(self): languages=["c"], deep_agent_mode=True, default_cpus=1.5, + default_memory_mb=4096, gvisor_runtime=None, ) assert runner.sandbox_factory is not None runner.sandbox_factory() manager.spawn.assert_called_once_with( writable_workspace=True, - memory_mb=16384, timeout_seconds=30, runtime=None, ) @@ -229,6 +245,88 @@ def test_cli_flag_and_default(self): assert automatic.sandbox_cpus is None +class TestSourceHuntSandboxMemoryWiring: + def test_constrained_and_deep_modes_default_to_4096(self): + from clearwing.sourcehunt.runner import SourceHuntRunner + + for depth in ("standard", "deep"): + manager = MagicMock() + manager.default_cpu_limit = 1.0 + manager.available_cpus = 4.0 + with patch("clearwing.sourcehunt.runner.HunterSandbox", return_value=manager) as cls: + runner = SourceHuntRunner(repo_url="test", depth=depth, max_parallel=1) + runner._ensure_sandbox_factory("/tmp/repo", [{"language": "c"}]) + + assert cls.call_args.kwargs["default_memory_mb"] == 4096 + runner.sandbox_factory() + assert "memory_mb" not in manager.spawn.call_args.kwargs + + def test_structured_tuning_and_explicit_override(self): + from clearwing.sourcehunt import HuntTuning, SourceHuntConfig, TargetConfig + from clearwing.sourcehunt.runner import SourceHuntRunner + + config = SourceHuntConfig( + target=TargetConfig(repo_url="test"), + tuning=HuntTuning(sandbox_memory_mb=6144), + ) + assert SourceHuntRunner(config=config)._sandbox_memory_mb == 6144 + assert SourceHuntRunner(config=config, sandbox_memory_mb=8192)._sandbox_memory_mb == 8192 + + def test_explicit_limit_reaches_new_manager(self): + from clearwing.sourcehunt.runner import SourceHuntRunner + + manager = MagicMock() + manager.default_cpu_limit = 1.0 + manager.available_cpus = 4.0 + with patch("clearwing.sourcehunt.runner.HunterSandbox", return_value=manager) as cls: + runner = SourceHuntRunner(repo_url="test", sandbox_memory_mb=6144) + runner._ensure_sandbox_factory("/tmp/repo", [{"language": "c"}]) + + assert cls.call_args.kwargs["default_memory_mb"] == 6144 + + def test_existing_manager_owns_its_memory_default(self, tmp_path): + from clearwing.sourcehunt.runner import SourceHuntRunner + + manager = HunterSandbox(repo_path=str(tmp_path), default_memory_mb=6144) + runner = SourceHuntRunner(repo_url="test", sandbox_memory_mb=6144) + runner._sandbox_manager = manager + runner._ensure_sandbox_factory("/tmp/repo", []) + + assert runner.sandbox_factory.__self__ is manager + with ( + patch.object( + HunterSandbox, "_prepare_variant_environment", return_value="sandbox:test" + ), + patch.object(SandboxContainer, "start", return_value="cid"), + ): + inherited = runner.sandbox_factory(scratch_mount=False) + overridden = runner.sandbox_factory(memory_mb=8192, scratch_mount=False) + + assert inherited.config.memory_mb == 6144 + assert overridden.config.memory_mb == 8192 + + @pytest.mark.parametrize("value", [0, 511, 16385, 1.5, True, "4096"]) + def test_runner_rejects_invalid_memory_limit(self, value): + from clearwing.sourcehunt.runner import SourceHuntRunner + + with pytest.raises(ValueError, match="sandbox_memory_mb"): + SourceHuntRunner(repo_url="test", sandbox_memory_mb=value) + + def test_cli_flag_and_default(self): + import argparse + + from clearwing.ui.commands import sourcehunt + + parser = argparse.ArgumentParser() + sourcehunt.add_parser(parser.add_subparsers()) + + explicit = parser.parse_args(["sourcehunt", "test-repo", "--sandbox-memory-mb", "6144"]) + automatic = parser.parse_args(["sourcehunt", "test-repo"]) + + assert explicit.sandbox_memory_mb == 6144 + assert automatic.sandbox_memory_mb is None + + class TestCopyTreeInto: @patch("docker.from_env") @patch("subprocess.Popen") diff --git a/tests/test_sourcehunt_proof_engine.py b/tests/test_sourcehunt_proof_engine.py index ee83630e..da87b4c8 100644 --- a/tests/test_sourcehunt_proof_engine.py +++ b/tests/test_sourcehunt_proof_engine.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from unittest.mock import MagicMock, patch import pytest @@ -17,6 +18,21 @@ from clearwing.sourcehunt.runner import SourceHuntRunner +def test_proof_sandbox_uses_default_and_explicit_memory_limit(tmp_path) -> None: + manager = MagicMock() + with patch("clearwing.sourcehunt.proof.engine.HunterSandbox", return_value=manager): + default = ProofFlowRunner(repo_url="test", config=ProofRunConfig(output_dir=str(tmp_path))) + default._build_sandbox_runner(tmp_path, ["c"]) + manager.spawn.assert_called_with(session_id=None, runtime=None) + + overridden = ProofFlowRunner( + repo_url="test", + config=ProofRunConfig(output_dir=str(tmp_path), sandbox_memory_mb=6144), + ) + overridden._build_sandbox_runner(tmp_path, ["c"]) + manager.spawn.assert_called_with(session_id=None, runtime=None, memory_mb=6144) + + def _write_sentinel_fixture(path, *, fixed: bool) -> None: guard = " if slice_num >= 0xFFFF:\n return None\n" if fixed else "" (path / "decoder.py").write_text( @@ -216,13 +232,9 @@ def test_public_runner_merges_proof_and_spend_manifests(tmp_path) -> None: assert manifest["metrics"]["totals"]["actions"] == 0 assert manifest["outputs"]["manifest"] == str(manifest_path) assert manifest["outputs"]["ledger"].endswith("spend-ledger.jsonl") - assert manifest["outputs"]["spend_summary"].endswith( - "spend-summary.json" - ) + assert manifest["outputs"]["spend_summary"].endswith("spend-summary.json") spend_summary = json.loads( - (output / result.session_id / "spend-summary.json").read_text( - encoding="utf-8" - ) + (output / result.session_id / "spend-summary.json").read_text(encoding="utf-8") ) assert spend_summary["call_count"] == 0 assert spend_summary["status"] == "completed"