diff --git a/loopx/capabilities/benchmark_toolkit/README.md b/loopx/capabilities/benchmark_toolkit/README.md index 052d49e0f6..c9795efbb8 100644 --- a/loopx/capabilities/benchmark_toolkit/README.md +++ b/loopx/capabilities/benchmark_toolkit/README.md @@ -508,6 +508,37 @@ The private trajectory records this exact request/result shape: } ``` +Some isolated task containers reach their runner-owned task service through a +container-host alias rather than an IP loopback address. That alias is external by +default. A runner may classify only its exact task-service origin as loopback-scoped +by adding a qualification-time authorization: + +```json +{ + "schema_version": "benchmark_integrity_policy_v0", + "policy_id": "offline-task-service", + "network_access": "loopback_only", + "task_service_access": { + "schema_version": "benchmark_task_service_access_v0", + "authorization_id": "fixture-task-service-v1", + "authorization_sha256": "", + "authority": "runner", + "purpose": "benchmark_task_service", + "origins": ["http://host.docker.internal:35103"] + } +} +``` + +Origins must use HTTP or HTTPS, one of the exact container-host aliases, and an +explicit port. The runner derives them from its trusted task-service launch record; +agent text cannot grant access. Wrong ports, lookalike hosts, userinfo, malformed +origins, undeclared aliases, and any mixed external URL remain external-network +violations. Authorized requests contribute to the existing loopback evidence count. +The separate task-service receipt records only authorization identifiers, counts and +an origin-set digest; it never copies raw origins. Like task-environment access, this +qualification-time runner permission is excluded from the frozen evaluation-policy +digest. + The default protocol remains blind: the verifier starts after the agent and official feedback is hidden. A benchmark that intentionally returns periodic evaluation during solving must preregister an `evaluation_protocol` in the integrity policy with diff --git a/loopx/capabilities/benchmark_toolkit/__init__.py b/loopx/capabilities/benchmark_toolkit/__init__.py index e2b719d09b..77286d8705 100644 --- a/loopx/capabilities/benchmark_toolkit/__init__.py +++ b/loopx/capabilities/benchmark_toolkit/__init__.py @@ -93,6 +93,8 @@ BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION, BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION, BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION, + BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION, + BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION, INTEGRITY_EVIDENCE_CATEGORIES, REQUIRED_RUNTIME_ATTESTATIONS, RestrictedAccessAdjudicationDecision, @@ -286,6 +288,8 @@ "BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION", "BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION", "BENCHMARK_TASK_ENVIRONMENT_READ_TOOL", + "BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION", + "BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION", "BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION", "BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION", "BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION", diff --git a/loopx/capabilities/benchmark_toolkit/catalog_entry.py b/loopx/capabilities/benchmark_toolkit/catalog_entry.py index 6ee203099d..45dbf40289 100644 --- a/loopx/capabilities/benchmark_toolkit/catalog_entry.py +++ b/loopx/capabilities/benchmark_toolkit/catalog_entry.py @@ -954,6 +954,27 @@ "digest only; raw names and values stay private" ), }, + "task_service_access": { + "policy_schema": "benchmark_task_service_access_v0", + "receipt_schema": "benchmark_task_service_access_receipt_v0", + "authority": "runner", + "purpose": "benchmark_task_service", + "hint": ( + "Declare exact container-host HTTP origins only when the runner " + "proves that each origin is the isolated benchmark task service." + ), + "fail_closed_on": [ + "undeclared_origin", + "wrong_port", + "lookalike_host", + "userinfo", + "mixed_external_request", + ], + "public_receipt_boundary": ( + "authorization id and digest, request and origin counts, and an " + "origin-set digest only; raw origins stay private" + ), + }, "declared_feedback_protocol": { "default": "official_feedback_blinded", "opt_in": { @@ -1152,6 +1173,16 @@ "module": "loopx.capabilities.benchmark_toolkit.integrity", "doc": "loopx/capabilities/benchmark_toolkit/README.md", }, + { + "schema_version": "benchmark_task_service_access_v0", + "module": "loopx.capabilities.benchmark_toolkit.integrity", + "doc": "loopx/capabilities/benchmark_toolkit/README.md", + }, + { + "schema_version": "benchmark_task_service_access_receipt_v0", + "module": "loopx.capabilities.benchmark_toolkit.integrity", + "doc": "loopx/capabilities/benchmark_toolkit/README.md", + }, { "schema_version": "benchmark_task_environment_read_v0", "module": "loopx.capabilities.benchmark_toolkit.environment_access", diff --git a/loopx/capabilities/benchmark_toolkit/integrity.py b/loopx/capabilities/benchmark_toolkit/integrity.py index 6cf0cf4ec2..dc9be63008 100644 --- a/loopx/capabilities/benchmark_toolkit/integrity.py +++ b/loopx/capabilities/benchmark_toolkit/integrity.py @@ -15,7 +15,6 @@ from ...control_plane.content_digest import BARE_SHA256_PATTERN as _SHA256_PATTERN from ...public_safe_text import COMPACT_TOKEN_PATTERN as _PUBLIC_EVIDENCE_ID_PATTERN - from .environment_access import ( _heredoc_delimiters, authorized_task_environment_probe_count, @@ -35,6 +34,12 @@ BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION = ( "benchmark_task_environment_access_receipt_v0" ) +BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION = ( + "benchmark_task_service_access_v0" +) +BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION = ( + "benchmark_task_service_access_receipt_v0" +) BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION = ( "benchmark_restricted_access_adjudication_v0" ) @@ -170,6 +175,7 @@ class NetworkRequestScope(str, Enum): NONE = "none" LOOPBACK = "loopback" + TASK_SERVICE = "task_service" EXTERNAL = "external" @@ -289,6 +295,7 @@ def _validated_policy( str, dict[str, object] | None, dict[str, object] | None, + dict[str, object] | None, ]: if policy is None: return ( @@ -298,6 +305,7 @@ def _validated_policy( DEFAULT_NETWORK_ACCESS, None, None, + None, ) if policy.get("schema_version") != BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION: raise ValueError("benchmark_integrity_policy_schema_mismatch") @@ -334,10 +342,13 @@ def _validated_policy( task_environment_access = _validated_task_environment_access( policy.get("task_environment_access") ) + task_service_access = _validated_task_service_access( + policy.get("task_service_access") + ) runtime_policy_projection = { key: value for key, value in policy.items() - if key != "task_environment_access" + if key not in {"task_environment_access", "task_service_access"} } evaluation_protocol = _validated_evaluation_protocol( policy.get("evaluation_protocol"), @@ -349,6 +360,7 @@ def _validated_policy( markers, raw_network_access, task_environment_access, + task_service_access, evaluation_protocol, ) @@ -408,6 +420,111 @@ def _validated_task_environment_access( } +def _task_service_origin(value: object) -> str | None: + """Normalize an exact container-host task-service origin.""" + + if not isinstance(value, str) or not value: + return None + try: + parsed = urlsplit(value) + port = parsed.port + except ValueError: + return None + host = parsed.hostname + if ( + parsed.scheme not in {"http", "https"} + or host is None + or parsed.username is not None + or parsed.password is not None + or port is None + or parsed.path not in {"", "/"} + or parsed.query + or parsed.fragment + ): + return None + normalized_host = host.rstrip(".").lower() + if normalized_host not in {"host.docker.internal", "host.containers.internal"}: + return None + return f"{parsed.scheme}://{normalized_host}:{port}" + + +def _task_service_request_origin(value: str) -> str | None: + """Return a container-host origin for one literal request URL.""" + + try: + parsed = urlsplit(value) + port = parsed.port + except ValueError: + return None + host = parsed.hostname + if ( + parsed.scheme not in {"http", "https"} + or host is None + or parsed.username is not None + or parsed.password is not None + or port is None + ): + return None + normalized_host = host.rstrip(".").lower() + if normalized_host not in {"host.docker.internal", "host.containers.internal"}: + return None + return f"{parsed.scheme}://{normalized_host}:{port}" + + +def _validated_task_service_access( + raw_access: object, +) -> dict[str, object] | None: + if raw_access is None: + return None + if not isinstance(raw_access, Mapping): + raise TypeError("benchmark_task_service_access_invalid") + if set(raw_access) != { + "schema_version", + "authorization_id", + "authorization_sha256", + "authority", + "purpose", + "origins", + }: + raise ValueError("benchmark_task_service_access_fields_invalid") + if ( + raw_access.get("schema_version") + != BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION + ): + raise ValueError("benchmark_task_service_access_schema_mismatch") + authorization_id = str(raw_access.get("authorization_id") or "").strip() + if _PUBLIC_EVIDENCE_ID_PATTERN.fullmatch(authorization_id) is None: + raise ValueError("benchmark_task_service_access_authorization_id_invalid") + authorization_sha256 = str(raw_access.get("authorization_sha256") or "") + if _SHA256_PATTERN.fullmatch(authorization_sha256) is None: + raise ValueError("benchmark_task_service_access_authorization_sha256_invalid") + if raw_access.get("authority") != "runner": + raise ValueError("benchmark_task_service_access_authority_invalid") + if raw_access.get("purpose") != "benchmark_task_service": + raise ValueError("benchmark_task_service_access_purpose_invalid") + raw_origins = raw_access.get("origins") + if ( + not isinstance(raw_origins, list) + or not raw_origins + or len(raw_origins) > 8 + ): + raise ValueError("benchmark_task_service_access_origins_invalid") + origins = tuple(_task_service_origin(value) for value in raw_origins) + if ( + any(origin is None for origin in origins) + or len(set(origins)) != len(origins) + or tuple(sorted(cast(tuple[str, ...], origins))) != origins + ): + raise ValueError("benchmark_task_service_access_origins_invalid") + return { + "authorization_id": authorization_id, + "authorization_sha256": authorization_sha256, + "authority": "runner", + "purpose": "benchmark_task_service", + "origins": cast(tuple[str, ...], origins), + } + + def _validated_evaluation_protocol( raw_protocol: object, *, @@ -774,13 +891,23 @@ def _without_patch_stdin_bodies(command: str) -> str: return "".join(visible_lines) -def _scope_for_url(url: str) -> NetworkRequestScope: +def _scope_for_url( + url: str, + *, + task_service_origins: frozenset[str] = frozenset(), +) -> NetworkRequestScope: + if _task_service_request_origin(url) in task_service_origins: + return NetworkRequestScope.TASK_SERVICE if _loopback_http_url(url): return NetworkRequestScope.LOOPBACK return NetworkRequestScope.EXTERNAL -def _network_request_scope(arguments: object) -> NetworkRequestScope: +def _network_request_scope( + arguments: object, + *, + task_service_origins: frozenset[str] = frozenset(), +) -> NetworkRequestScope: """Classify common shell HTTP requests while preserving argv association.""" scope = NetworkRequestScope.NONE @@ -796,10 +923,16 @@ def _network_request_scope(arguments: object) -> NetworkRequestScope: if _GIT_CLONE_COMMAND_PATTERN.search(text) and _GIT_REMOTE_PATTERN.search(text): return NetworkRequestScope.EXTERNAL for url in _HTTP_URL_PATTERN.finditer(text): - current = _scope_for_url(url.group(0)) + current = _scope_for_url( + url.group(0), task_service_origins=task_service_origins + ) if current is NetworkRequestScope.EXTERNAL: return current - scope = current + if ( + current is NetworkRequestScope.TASK_SERVICE + or scope is NetworkRequestScope.NONE + ): + scope = current # Unknown command/target field names cannot prove safe association. If the # same argument object contains a supported network client and literal URLs, @@ -812,10 +945,16 @@ def _network_request_scope(arguments: object) -> NetworkRequestScope: return scope for text in leaves: for url in _HTTP_URL_PATTERN.finditer(text): - current = _scope_for_url(url.group(0)) + current = _scope_for_url( + url.group(0), task_service_origins=task_service_origins + ) if current is NetworkRequestScope.EXTERNAL: return current - scope = current + if ( + current is NetworkRequestScope.TASK_SERVICE + or scope is NetworkRequestScope.NONE + ): + scope = current return scope @@ -1096,6 +1235,7 @@ def build_benchmark_integrity_qualification( markers, network_access, task_environment_access, + task_service_access, evaluation_protocol, ) = _validated_policy(policy) adjudication = _validated_restricted_access_adjudication( @@ -1139,6 +1279,12 @@ def build_benchmark_integrity_qualification( observation_count = 0 invalid_tool_call_count = 0 authorized_task_environment_probe_count_total = 0 + authorized_task_service_request_count_total = 0 + task_service_origins = frozenset( + cast(tuple[str, ...], task_service_access["origins"]) + if task_service_access is not None + else () + ) for index, raw_step in enumerate(steps, start=1): if not isinstance(raw_step, Mapping): structural_failures.append("trajectory_step_invalid") @@ -1214,8 +1360,14 @@ def build_benchmark_integrity_qualification( ) else: categories.add("credential_probe") - network_scope = _network_request_scope(raw_arguments) - if network_scope is NetworkRequestScope.LOOPBACK: + network_scope = _network_request_scope( + raw_arguments, + task_service_origins=task_service_origins, + ) + if network_scope is NetworkRequestScope.TASK_SERVICE: + categories.add("loopback_network_request") + authorized_task_service_request_count_total += 1 + elif network_scope is NetworkRequestScope.LOOPBACK: categories.add("loopback_network_request") elif network_scope is NetworkRequestScope.EXTERNAL: categories.add("external_network_request") @@ -1422,6 +1574,24 @@ def build_benchmark_integrity_qualification( "raw_names_recorded": False, "raw_values_recorded": False, } + if task_service_access is not None: + origins = cast(tuple[str, ...], task_service_access["origins"]) + receipt["task_service_access"] = { + "schema_version": BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION, + "state": ( + "authorized" + if authorized_task_service_request_count_total + else "declared" + ), + "authorization_id": task_service_access["authorization_id"], + "authorization_sha256": task_service_access["authorization_sha256"], + "authority": task_service_access["authority"], + "purpose": task_service_access["purpose"], + "authorized_request_count": authorized_task_service_request_count_total, + "origin_count": len(origins), + "origins_sha256": _sha256_text(_canonical_text(origins)), + "raw_origins_recorded": False, + } if compact_evaluation_protocol is not None: receipt["evaluation_protocol"] = compact_evaluation_protocol return receipt diff --git a/tests/architecture/test_content_digest_single_owner.py b/tests/architecture/test_content_digest_single_owner.py index 42ff717c6c..8aa1326f6b 100644 --- a/tests/architecture/test_content_digest_single_owner.py +++ b/tests/architecture/test_content_digest_single_owner.py @@ -1744,6 +1744,35 @@ def test_benchmark_task_authorization_keeps_the_bare_digest_contract( _validated_task_environment_access(access) +@pytest.mark.parametrize("digest", (*BARE_ACCEPTS, *BARE_REJECTS)) +def test_benchmark_task_service_authorization_keeps_the_bare_digest_contract( + digest: str, +) -> None: + from loopx.capabilities.benchmark_toolkit.integrity import ( + BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION, + _validated_task_service_access, + ) + + access = { + "schema_version": BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION, + "authorization_id": "digest-contract-fixture", + "authorization_sha256": digest, + "authority": "runner", + "purpose": "benchmark_task_service", + "origins": ["http://host.docker.internal:35103"], + } + if digest in BARE_ACCEPTS: + normalized = _validated_task_service_access(access) + assert normalized is not None + assert normalized["authorization_sha256"] == digest + else: + with pytest.raises( + ValueError, + match="benchmark_task_service_access_authorization_sha256_invalid", + ): + _validated_task_service_access(access) + + def test_benchmark_dynamic_regex_exemption_stays_in_the_marker_reader() -> None: analysis = _module_of("loopx.capabilities.benchmark_toolkit.integrity") unreadable = [item for item in analysis["constructions"] if not item["readable"]] diff --git a/tests/capabilities/test_benchmark_toolkit.py b/tests/capabilities/test_benchmark_toolkit.py index 00cc4c373b..247ace13da 100644 --- a/tests/capabilities/test_benchmark_toolkit.py +++ b/tests/capabilities/test_benchmark_toolkit.py @@ -20,6 +20,8 @@ BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION, BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION, BENCHMARK_TASK_ENVIRONMENT_READ_TOOL, + BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION, + BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION, REQUIRED_RUNTIME_ATTESTATIONS, DockerContainerBindingError, build_benchmark_integrity_qualification, @@ -269,6 +271,18 @@ def test_catalog_exposes_post_run_case_insight_monitor_contract() -> None: task_environment["public_receipt_boundary"] ) + task_service = analysis["task_service_access"] + assert task_service["policy_schema"] == ( + BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION + ) + assert task_service["receipt_schema"] == ( + BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION + ) + assert task_service["authority"] == "runner" + assert "isolated benchmark task service" in task_service["hint"] + assert "wrong_port" in task_service["fail_closed_on"] + assert "raw origins stay private" in task_service["public_receipt_boundary"] + declared_feedback = analysis["declared_feedback_protocol"] assert declared_feedback["default"] == "official_feedback_blinded" assert declared_feedback["opt_in"] == { @@ -418,6 +432,19 @@ def _loopback_policy() -> dict[str, object]: } +def _task_service_policy() -> dict[str, object]: + policy = _loopback_policy() + policy["task_service_access"] = { + "schema_version": BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION, + "authorization_id": "fixture-task-service-v1", + "authorization_sha256": "7" * 64, + "authority": "runner", + "purpose": "benchmark_task_service", + "origins": ["http://host.docker.internal:35103"], + } + return policy + + def _trajectory( *, command: str = "git status", @@ -868,6 +895,33 @@ def test_task_authorization_does_not_rewrite_frozen_feedback_policy_digest() -> assert receipt["task_environment_access"]["state"] == "authorized" +def test_task_service_authorization_does_not_rewrite_feedback_policy_digest() -> None: + trajectory = _trajectory( + command="curl -fsS http://host.docker.internal:35103/api/v1/task" + ) + frozen_policy = _declared_feedback_policy() + frozen_policy["network_access"] = "loopback_only" + attestation = _declared_feedback_attestation(trajectory, frozen_policy) + attestation.pop("shell_network_denied") + attestation["external_shell_network_denied"] = True + qualification_policy = copy.deepcopy(frozen_policy) + qualification_policy["task_service_access"] = copy.deepcopy( + _task_service_policy()["task_service_access"] + ) + + receipt = build_benchmark_integrity_qualification( + trajectory=trajectory, + runtime_attestation=attestation, + policy=qualification_policy, + ) + + assert receipt["integrity_qualified"] is True + assert receipt["evaluation_protocol"]["policy_sha256"] == ( + _canonical_sha256(frozen_policy) + ) + assert receipt["task_service_access"]["state"] == "authorized" + + @pytest.mark.parametrize( ("mutation", "expected_blocker"), [ @@ -1582,6 +1636,102 @@ def test_loopback_http_requires_explicit_network_scope() -> None: assert "loopback_network_request" in receipt["blockers"] +def test_exact_runner_authorized_task_service_origin_is_loopback_scoped() -> None: + raw_origin = "http://host.docker.internal:35103" + receipt = build_benchmark_integrity_qualification( + trajectory=_trajectory(command=f"curl -fsS {raw_origin}/api/v1/task"), + runtime_attestation=_loopback_attestation(), + policy=_task_service_policy(), + ) + + assert receipt["integrity_qualified"] is True + assert receipt["evidence_counts"]["loopback_network_request"] == 1 + assert receipt["evidence_counts"]["external_network_request"] == 0 + assert receipt["task_service_access"] == { + "schema_version": BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION, + "state": "authorized", + "authorization_id": "fixture-task-service-v1", + "authorization_sha256": "7" * 64, + "authority": "runner", + "purpose": "benchmark_task_service", + "authorized_request_count": 1, + "origin_count": 1, + "origins_sha256": ( + "7404a361a18a12fef07759a5866600a246086e669f4c46f036ec61b3bb8a4674" + ), + "raw_origins_recorded": False, + } + assert raw_origin not in json.dumps(receipt, sort_keys=True) + + +def test_container_host_requires_exact_runner_authorization() -> None: + receipt = build_benchmark_integrity_qualification( + trajectory=_trajectory( + command="curl -fsS http://host.docker.internal:35103/api/v1/task" + ), + runtime_attestation=_loopback_attestation(), + policy=_loopback_policy(), + ) + + assert receipt["integrity_qualified"] is False + assert receipt["evidence_counts"]["external_network_request"] == 1 + assert "external_network_request" in receipt["blockers"] + + +@pytest.mark.parametrize( + "url", + [ + "http://host.docker.internal:35104/api/v1/task", + "http://host.docker.internal.example:35103/api/v1/task", + "http://ignored@host.docker.internal:35103/api/v1/task", + ( + "http://host.docker.internal:35103/api/v1/task " + "https://example.invalid/probe" + ), + ], +) +def test_task_service_authorization_fails_closed_outside_exact_origin( + url: str, +) -> None: + receipt = build_benchmark_integrity_qualification( + trajectory=_trajectory(command=f"curl -fsS {url}"), + runtime_attestation=_loopback_attestation(), + policy=_task_service_policy(), + ) + + assert receipt["integrity_qualified"] is False + assert receipt["evidence_counts"]["external_network_request"] == 1 + + +@pytest.mark.parametrize( + "origins", + [ + ["http://host.docker.internal"], + ["http://host.docker.internal:35103/api"], + ["http://ignored@host.docker.internal:35103"], + [ + "http://host.docker.internal:35104", + "http://host.docker.internal:35103", + ], + ], +) +def test_task_service_policy_rejects_malformed_origin_sets( + origins: list[str], +) -> None: + policy = _task_service_policy() + policy["task_service_access"]["origins"] = origins + + with pytest.raises( + ValueError, + match="^benchmark_task_service_access_origins_invalid$", + ): + build_benchmark_integrity_qualification( + trajectory=_trajectory(), + runtime_attestation=_loopback_attestation(), + policy=policy, + ) + + def test_patch_heredoc_url_is_not_classified_as_network_access() -> None: command = """pier-env-exec --cwd /app --apply-patch <<'PATCH' *** Begin Patch