diff --git a/docs/reference/protocols/loopx-turn-v0.md b/docs/reference/protocols/loopx-turn-v0.md index 53c0d17ad4..07e33bf7a1 100644 --- a/docs/reference/protocols/loopx-turn-v0.md +++ b/docs/reference/protocols/loopx-turn-v0.md @@ -344,6 +344,15 @@ boundary. Caller death can precede cleanup; the local lane lock alone cannot certify no overlap with a replacement executor. Authority-bound renewal, revocation and uncertain-effect recovery remain a separate delivery. +The Codex CLI and managed app-server operation hosts use a strict wire schema: all +properties are required, and work/validation evidence is an array or null. Stop +results send both evidence fields as null; the Codex boundary removes only those +null placeholders before internal validation. Material results with an +`execution_profile` must supply both evidence arrays; null or missing evidence is +rejected at that boundary. Empty or malformed arrays still fail the existing +executor validator. Legacy material results without a profile may omit evidence. +Other host contracts are unchanged. + ### Repeatable Codex CLI Qualification The repository includes an opt-in end-to-end qualification that creates an diff --git a/examples/loopx-turn-codex-cli-e2e-smoke.py b/examples/loopx-turn-codex-cli-e2e-smoke.py index d88318fc39..c25a3f06e8 100755 --- a/examples/loopx-turn-codex-cli-e2e-smoke.py +++ b/examples/loopx-turn-codex-cli-e2e-smoke.py @@ -147,6 +147,7 @@ def _write_fake_codex(root: Path) -> Path: raise SystemExit("unexpected marker value") turn_number = int(match.group(1)) + 1 marker.write_text({MARKER_PREFIX!r} + str(turn_number), encoding="utf-8") +assert marker.read_text(encoding="utf-8") == {MARKER_PREFIX!r} + str(turn_number) print(json.dumps({{ "type": "thread.started", "thread_id": "session-fixture-0001", @@ -164,6 +165,8 @@ def _write_fake_codex(root: Path) -> Path: "delivery_outcome": "outcome_progress", "vision_unchanged_reason": "The fixture objective remains unchanged.", "summary": f"The isolated public marker reached step {{turn_number}}.", + "work_evidence": ["artifact:turn-e2e-marker"], + "validation_evidence": ["check:marker-file-readback"], }}), encoding="utf-8") """, ) diff --git a/loopx/control_plane/turn_driver/codex_cli.py b/loopx/control_plane/turn_driver/codex_cli.py index b36c8d2437..b6026d6425 100644 --- a/loopx/control_plane/turn_driver/codex_cli.py +++ b/loopx/control_plane/turn_driver/codex_cli.py @@ -37,6 +37,8 @@ HOST_RESULT_EVIDENCE_LIMITS, HOST_RESULT_TEXT_LIMITS, LOOPX_TURN_HOST_REQUEST_SCHEMA_VERSION, + MATERIAL_HOST_RESULT_KINDS, + STOP_HOST_RESULT_KINDS, ) from ..capability_hooks import capability_result_fields from .execution_profile import require_supported_reasoning_effort @@ -263,8 +265,9 @@ def codex_cli_result_schema( HOST_RESULT_EVIDENCE_LIMITS.items() ): properties[evidence_field] = { - "type": "array", + "type": ["array", "null"], "items": {"type": "string", "maxLength": max_item_chars}, + "minItems": 1, "maxItems": max_items, } properties.update(capability_result_fields(request)) @@ -276,34 +279,37 @@ def codex_cli_result_schema( "enum": ["codex-cli"], } properties["child_execution_receipts"] = child_receipts - envelope = _mapping(request.get("turn_envelope")) if request else {} - capsule = _mapping(envelope.get("contract_capsule")) - profile = capsule.get("execution_profile") - evidence_required = isinstance(profile, Mapping) and bool(profile) - schema: dict[str, Any] = { + # Codex strict output requires every property and forbids conditional schemas. + return { "type": "object", "properties": properties, - "required": [ - field for field in properties - if field not in HOST_RESULT_EVIDENCE_LIMITS - ], + "required": list(properties), "additionalProperties": False, } - if evidence_required: - schema["allOf"] = [ - { - "if": { - "properties": { - "result_kind": {"enum": list(CODEX_CLI_RESULT_KINDS[:3])} - }, - "required": ["result_kind"], - }, - "then": { - "required": list(HOST_RESULT_EVIDENCE_LIMITS), - }, - } - ] - return schema + + +def normalize_codex_result( + request: Mapping[str, Any], result: Mapping[str, Any], +) -> dict[str, Any]: + """Translate strict-output null placeholders at both Codex boundaries.""" + normalized = dict(result) + kind = normalized.get("result_kind") + if not isinstance(kind, str): + return normalized + if kind in STOP_HOST_RESULT_KINDS: + for field in HOST_RESULT_EVIDENCE_LIMITS: + if normalized.get(field) is None: + normalized.pop(field, None) + elif kind in MATERIAL_HOST_RESULT_KINDS: + envelope = _mapping(request.get("turn_envelope")) + profile = _mapping(envelope.get("contract_capsule")).get("execution_profile") + if isinstance(profile, Mapping) and profile and any( + normalized.get(field) is None for field in HOST_RESULT_EVIDENCE_LIMITS + ): + raise BuiltInHostError("codex_material_result_missing_evidence") + # Preserve non-null evidence, including malformed/empty arrays, for the + # existing executor validator. Legacy material results may omit evidence. + return normalized def _prompt(request: Mapping[str, Any]) -> str: @@ -330,7 +336,7 @@ def _prompt(request: Mapping[str, Any]) -> str: + "Read work_context and remaining required_reads before primary_action; do not repeat fulfilled reads; perform work only when its contract allows it. The selected Todo summary is not its full requirements, and completing that work does not prove Goal completion.", "Do not write LoopX state, spend quota, or apply scheduler changes; the adapter owns those effects.", "Return only the schema-constrained result. For validated_progress, repair_required, or replan_required, fill every material field with public-safe evidence.", - "For those material results, preserve the actual delivery_batch_scale and delivery_outcome selected from the signed execution_profile's work and outcome requirements. Return work_evidence and validation_evidence as separate arrays of short public-safe strings; name the artifact or observed work in the first and the independent command or readback in the second. Omit both arrays for stop results.", + "For those material results, preserve the actual delivery_batch_scale and delivery_outcome selected from the signed execution_profile's work and outcome requirements. Return work_evidence and validation_evidence as separate non-empty arrays of short public-safe strings; name the artifact or observed work in the first and the independent command or readback in the second. For legacy requests without execution_profile, use null when evidence is not supplied. For stop results, set both evidence fields to null.", "For those material results, set path_delta_mode=material_replan only when this Turn changes a prior assumption, route, scope, acceptance rule, or stops prior work; then provide a complete bounded agent vision packet with goal_path_delta_v0 in agent_vision_json and leave vision_unchanged_reason empty.", "For routine continuation, retry, successor creation, or no-change replanning, set path_delta_mode=unchanged, leave agent_vision_json empty, and provide vision_unchanged_reason.", "When replan_action_packet requires a semantic delta, return progress_observation_json as a JSON-encoded typed_progress_observation_v0 with result_class, the observed surface_id/hypothesis_id/probe_kind or blocker/coverage fields, and opaque evidence_ids. Report only independently checked evidence; a plan summary or a vision change alone cannot substitute for the required observation. Otherwise leave this field empty.", @@ -943,4 +949,4 @@ def observe_stderr(line: str) -> None: raise BuiltInHostError("codex_cli_final_result_missing") from exc if not isinstance(result, dict): raise BuiltInHostError("codex_cli_final_result_not_object") - return result + return normalize_codex_result(request, result) diff --git a/loopx/control_plane/turn_driver/codex_operation_host.py b/loopx/control_plane/turn_driver/codex_operation_host.py index 1869ee1635..6e6a82f01e 100644 --- a/loopx/control_plane/turn_driver/codex_operation_host.py +++ b/loopx/control_plane/turn_driver/codex_operation_host.py @@ -23,6 +23,7 @@ _prompt, normalize_codex_stdio_mcp_server, codex_cli_result_schema, + normalize_codex_result, ) from .codex_sessions import _lineage, select_managed_operation_session from .executor import LOOPX_TURN_HOST_REQUEST_SCHEMA_VERSION @@ -218,7 +219,7 @@ def record_child() -> None: turn_start_guard=lambda: launch_scope("codex_operation_turn_start"), on_native_item=observe_child if child_observer is not None else None, ) - return result + return normalize_codex_result(request, result) except CodexChatAgentError as exc: raise BuiltInHostError( "codex_operation_host_" + exc.error_code, diff --git a/skills/loopx-self-repair/references/repair-patterns.md b/skills/loopx-self-repair/references/repair-patterns.md index adf5f9f384..2f02d796ee 100644 --- a/skills/loopx-self-repair/references/repair-patterns.md +++ b/skills/loopx-self-repair/references/repair-patterns.md @@ -6,6 +6,7 @@ teaches a reusable control-plane lesson. | Pattern | Symptoms | Evidence To Read | Likely Root | Durable Repair | | --- | --- | --- | --- | --- | | `host_bridge_reader_orphan` | A rejected bridge observation leaves its reader thread and output pipe alive after the child has exited. | Original framing error, cleanup result, child exit, bounded-queue backpressure and reader/pipe state across repeated failures. | Cancellation rejects another malformed frame; the sole consumer leaves while the producer blocks on a full queue, and joining alone cannot release it. | Stop publication, release the waiting producer, join and close through the existing transport helper. Retain cleanup-unconfirmed results and original Host drain ownership; reclaiming a reader does not prove Host exit. Cover malformed JSON and invalid typed observations with real subprocesses, plus existing real Host cancellation and success paths. Do not increase timeouts or fabricate a successful result. | +| `codex_strict_result_schema_gap` | A Codex Turn fails with invalid_json_schema before execution, or a strict-schema stop result fails internal evidence validation. | Generated Codex schema, required properties, unsupported conditional keywords, and both CLI/app-server result boundaries. | Optional evidence and conditional schema requirements conflict with strict Structured Outputs; internal stop results prohibit evidence fields. | Require nullable evidence fields on the Codex wire and remove unsupported conditionals. Normalize only stop null placeholders in both Codex transports; reject missing/null profile material evidence and preserve non-null evidence for the existing validator. Cover legacy absence, invalid arrays and an isolated schema-acceptance probe without changing other hosts. | | `initial_preference_view_omission` | A fulfilled preference hook leaves corrections, retirement or advisory instructions absent from the recommended initial read. | The preference owner's complete current scoped view, actual selected reader/native input, empty and unavailable states, and same-capture continuation/recovery. | A hand-picked projection uses incomplete or stale child field names after the owner has already delivered the current view. | Reuse the complete owned current view in the existing initial selector; retain provenance, advisory authority, exact-scope freshness and immutable capture checks. Cover real File/SQLite lifecycle reads, empty replacements, unknown values and native consumer readback. Preserve changed-profile refusal and fresh recovery; disclose restored bytes as information cost, not compression or model benefit. | | `immutable_reader_parent_alias` | A warm native view refuses an unchanged reader reference after a verified capture read normalizes its parent path. | Cold descriptor bytes, warm baseline path and full digest, parent aliases versus distinct leaf names, and actual CLI recovery after compaction. | The reader accepts normal parent aliases, but immutable publication compares their literal path spellings. | Compare only equivalent parent spellings at the existing IO owner, retaining the exact leaf and every other metadata value; never rewrite the descriptor. Preserve full-digest, selection, permission and final-symlink checks. Cover both publication orders, distinct files and hardlinks, cold/warm native publication and registry-free current recovery; keep transport repair separate from retention and net-benefit claims. | | `todo_read_source_authority_label` | A canonical Todo read reports one provider in `authority_read` but labels the public result or projection source as File. | The same read's canonical provider revision, public source, envelope sources and Markdown across full, filtered, exact and compact views. | The shared formatter retained a provider-specific constant after provider routing moved to the canonical reader. | Reuse the existing canonical source authority for presentation and provenance. Exercise real CLI File/SQLite reads, missing results and source failure/recovery; preserve filtering, read-only behavior and fail-closed routing. A display correction does not select, migrate or qualify a provider. | diff --git a/tests/control_plane/test_reward_memory_host_isolation.py b/tests/control_plane/test_reward_memory_host_isolation.py index 5d31e72fff..cc0fd0c5c4 100644 --- a/tests/control_plane/test_reward_memory_host_isolation.py +++ b/tests/control_plane/test_reward_memory_host_isolation.py @@ -381,7 +381,9 @@ def test_real_turn_cli_memory_projection_and_settlement(tmp_path: Path, monkeypa 'classification': 'fixture_progress', 'recommended_action': 'Continue the original fixture', 'next_action': 'Run the next bounded fixture', 'delivery_batch_scale': 'single_surface', 'delivery_outcome': 'outcome_progress', 'vision_unchanged_reason': 'The original fixture remains open.', - 'path_delta_mode': 'unchanged', 'agent_vision_json': '', 'summary': 'Independent fixture validation passed.'} + 'path_delta_mode': 'unchanged', 'agent_vision_json': '', 'summary': 'Independent fixture validation passed.', + 'work_evidence': ['artifact:fixture-host-capture'], + 'validation_evidence': ['check:fixture-independent-validator']} # Simulate an older/custom host returning a memory field even when the current # result schema does not request it. The real adapter must isolate that input. result['reward_memory_reflection_json'] = '{"status":"no_evidence"}' diff --git a/tests/test_codex_operation_host.py b/tests/test_codex_operation_host.py index 535ffc4dff..5a677c033a 100644 --- a/tests/test_codex_operation_host.py +++ b/tests/test_codex_operation_host.py @@ -85,6 +85,7 @@ def emit(value): assert row["result"]["success"] is True answer = {name: "" for name, schema in properties.items() if schema["type"] == "string"} answer.update({"schema_version": "loopx_turn_result_v0", "turn_key": key, "result_kind": "wait", "completed_phases": ["host_execute", "typed_result"], "classification": "awaiting_operation_confirmation", "recommended_action": "Wait for actual human approval", "summary": "Owned native context retrieved; no approval or effect"}) + answer.update(work_evidence=None, validation_evidence=None) emit({"method": "item/agentMessage/delta", "params": {"threadId": thread, "turnId": turn, "delta": json.dumps(answer)}}) emit({"method": "turn/completed", "params": {"threadId": thread, "turn": {"id": turn, "status": "completed"}}}) """ diff --git a/tests/test_dsh_goal_mode.py b/tests/test_dsh_goal_mode.py index 118e4fccbe..a8a9b1cfed 100644 --- a/tests/test_dsh_goal_mode.py +++ b/tests/test_dsh_goal_mode.py @@ -16,7 +16,7 @@ turn_envelope_action_signature_document, ) from loopx.control_plane.turn_driver.executor import validate_loopx_turn_host_result -from loopx.control_plane.turn_driver.codex_cli import codex_cli_result_schema +from loopx.control_plane.turn_driver.codex_cli import codex_cli_result_schema, normalize_codex_result from loopx.control_plane.turn_driver.host_failure import ( BuiltInHostError, build_host_failure_record, @@ -733,16 +733,17 @@ def test_stop_candidates_cannot_declare_work_or_validation_evidence(kind: str) - @pytest.mark.parametrize("kind", ["wait", "user_action_required", "iteration_failed"]) -def test_codex_schema_stop_with_required_empty_path_fields_passes_executor(kind: str) -> None: - request = _signed_request() +@pytest.mark.parametrize("with_profile", [False, True]) +@pytest.mark.parametrize("evidence", [None, [], False, ["artifact:unexpected"]]) +def test_codex_schema_stop_with_required_empty_path_fields_passes_executor(kind, with_profile, evidence) -> None: + request = _signed_request(execution_profile={"work_kind": "engineering_change"} if with_profile else None) schema = codex_cli_result_schema(request) required = schema["required"] properties = schema["properties"] assert "path_delta_mode" in required and "agent_vision_json" in required assert properties["path_delta_mode"]["type"] == "string" assert properties["agent_vision_json"]["type"] == "string" - assert "work_evidence" not in required - assert "validation_evidence" not in required + assert "work_evidence" in required and "validation_evidence" in required value = {field: "" for field in required} value.update( { @@ -752,40 +753,47 @@ def test_codex_schema_stop_with_required_empty_path_fields_passes_executor(kind: "completed_phases": ["host_execute", "typed_result"], "classification": "bounded stop", "next_action": "Await the next authorized iteration.", + "work_evidence": evidence, + "validation_evidence": evidence, } ) - verdict = validate_loopx_turn_host_result(_turn_plan(request), value) - assert verdict["ok"], verdict["errors"] + adapted = normalize_codex_result(request, value) + verdict = validate_loopx_turn_host_result(_turn_plan(request), adapted) + assert verdict["ok"] is (evidence is None), verdict["errors"] + # Only null placeholders disappear; real/invalid evidence remains visible. + assert value["work_evidence"] is evidence + if evidence is not None: + assert adapted == value + else: + assert "work_evidence" not in adapted and "validation_evidence" not in adapted -def test_codex_schema_requires_profile_evidence_only_for_material_results() -> None: - request = _signed_request( - execution_profile={"minimum_scale": "multi_surface_or_implementation"} - ) - schema = codex_cli_result_schema(request) - - assert "work_evidence" not in schema["required"] - assert "validation_evidence" not in schema["required"] - assert schema["allOf"] == [ - { - "if": { - "properties": { - "result_kind": { - "enum": [ - "validated_progress", - "repair_required", - "replan_required", - ] - } - }, - "required": ["result_kind"], - }, - "then": { - "required": ["work_evidence", "validation_evidence"], - }, - } - ] +@pytest.mark.parametrize("kind", ["validated_progress", "repair_required", "replan_required"]) +@pytest.mark.parametrize("with_profile", [False, True]) +@pytest.mark.parametrize("field", ["work_evidence", "validation_evidence"]) +@pytest.mark.parametrize("evidence", ["missing", None, [], False, {}, ["artifact:checked"]]) +def test_codex_material_evidence_keeps_profile_and_legacy_rules(kind, with_profile, field, evidence): + request = _signed_request(execution_profile={"work_kind": "engineering_change"} if with_profile else None) + value = turn_host_adapter.build_result(request, { + "result_kind": kind, "classification": "artifact_checked", + "next_action": "Continue the accepted task.", + "vision_unchanged_reason": "The accepted Goal remains unchanged.", + "work_evidence": ["artifact:checked"], "validation_evidence": ["check:independent"], + }) + if evidence == "missing": + value.pop(field) + else: + value[field] = evidence + if with_profile and (evidence is None or evidence == "missing"): + with pytest.raises(BuiltInHostError, match="codex_material_result_missing_evidence"): + normalize_codex_result(request, value) + return + adapted = normalize_codex_result(request, value) + assert adapted == value + verdict = validate_loopx_turn_host_result(_turn_plan(request), adapted) + expected = evidence is None or evidence == "missing" or isinstance(evidence, list) and bool(evidence) + assert verdict["ok"] is expected, verdict["errors"] def _turn_plan(request: dict) -> dict: diff --git a/tests/test_loopx_turn_codex_cli.py b/tests/test_loopx_turn_codex_cli.py index 4846eb346c..341ef4ca2f 100644 --- a/tests/test_loopx_turn_codex_cli.py +++ b/tests/test_loopx_turn_codex_cli.py @@ -194,7 +194,7 @@ def _fake_codex(tmp_path: Path) -> tuple[Path, Path]: if os.environ.get("FAKE_CODEX_SLEEP"): time.sleep(float(os.environ["FAKE_CODEX_SLEEP"])) output_path = pathlib.Path(args[args.index("--output-last-message") + 1]) -output_path.write_text(json.dumps({ +result = { "schema_version": "loopx_turn_result_v0", "turn_key": turn_key, "result_kind": "validated_progress", @@ -208,7 +208,9 @@ def _fake_codex(tmp_path: Path) -> tuple[Path, Path]: "summary": "One public fixture advanced.", "work_evidence": ["artifact:fixture-change"], "validation_evidence": ["check:fixture-validation"] -}), encoding="utf-8") +} +result.update(json.loads(os.environ.get("FAKE_CODEX_RESULT_FIELDS", "{}"))) +output_path.write_text(json.dumps(result), encoding="utf-8") """ executable = write_python_host_fixture( executable, @@ -256,13 +258,45 @@ def _capture_fake_codex_failure( return exc_info.value, runtime_root -def test_codex_cli_result_schema_requires_only_bounded_contract_fields() -> None: - schema = codex_cli_result_schema() +@pytest.mark.parametrize("kind", ["wait", "user_action_required", "iteration_failed"]) +@pytest.mark.parametrize("with_profile", [False, True]) +def test_codex_cli_stop_null_evidence_reaches_executor(tmp_path, monkeypatch, kind, with_profile): + from loopx.control_plane.turn_driver.executor import validate_loopx_turn_host_result + + executable, log_path = _fake_codex(tmp_path) + monkeypatch.setenv("FAKE_CODEX_LOG", str(log_path)) + monkeypatch.setenv("FAKE_CODEX_RESULT_FIELDS", json.dumps({ + "result_kind": kind, "work_evidence": None, "validation_evidence": None, + "delivery_batch_scale": "", "delivery_outcome": "", "vision_unchanged_reason": "", + "path_delta_mode": "", "agent_vision_json": "", "progress_observation_json": "", + })) + request = _request() + if with_profile: + request["turn_envelope"]["contract_capsule"] = { + "execution_profile": {"work_kind": "engineering_change"} + } + result = run_codex_cli_host(request, runtime_root=tmp_path / "runtime", + project=tmp_path, codex_bin=str(executable), timeout_seconds=5) + assert "work_evidence" not in result and "validation_evidence" not in result + verdict = validate_loopx_turn_host_result({"transaction": {"turn_key": request["turn_key"]}}, result) + assert verdict["ok"], verdict["errors"] + + +@pytest.mark.parametrize("with_profile", [False, True]) +def test_codex_cli_result_schema_requires_only_bounded_contract_fields(with_profile) -> None: + request = _request() + if with_profile: + request["turn_envelope"]["contract_capsule"] = { + "execution_profile": {"work_kind": "engineering_change"} + } + schema = codex_cli_result_schema(request) assert schema["additionalProperties"] is False - assert set(schema["required"]) == ( - set(schema["properties"]) - {"work_evidence", "validation_evidence"} - ) + assert set(schema["required"]) == set(schema["properties"]) + assert not {"allOf", "if", "then"}.intersection(schema) + for field in ("work_evidence", "validation_evidence"): + assert schema["properties"][field]["type"] == ["array", "null"] + assert schema["properties"][field]["minItems"] == 1 assert "raw_trajectory" not in schema["properties"] assert "stdout" not in schema["properties"] assert "child_execution_receipts" not in schema["properties"]