From 94d86ea3a0d8bc504772e89c52a2c6208d484154 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 14:43:42 +0000 Subject: [PATCH 1/4] fix(ai): rebase adaptive defaults onto current main Keep ADR-0013 and drop the colliding ADR-0005 copies. Record one Unreleased changelog entry, restore the runtime-adapter and post-evaluation transport regressions, and correct the leftover post-chat docstring so it no longer describes a forced route. Co-authored-by: Seongho Bae --- CHANGELOG.md | 10 ++++ lineageweave/post_chat.py | 2 +- tests/test_adaptive_orchestrator_default.py | 49 +++++++++++++++++++ ..._contextual_orchestrator_default_policy.py | 40 +++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 tests/test_adaptive_orchestrator_default.py create mode 100644 tests/test_contextual_orchestrator_default_policy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0096828a2..709e76880 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- Product LLM adapters now request contextual-orchestrator + `mode="auto"` rather than forcing a one-model route. The + orchestrator owns the quality-sufficient route, verification, or + conducted workflow; the explicit adjudication `verify` contract + remains unchanged. + ## [0.71.0] - 2026-08-14 ### Added diff --git a/lineageweave/post_chat.py b/lineageweave/post_chat.py index 7ec67e943..d23624bc1 100644 --- a/lineageweave/post_chat.py +++ b/lineageweave/post_chat.py @@ -187,7 +187,7 @@ class ContextualOrchestratorPostChatClient: ``mode="verify"`` exists for (one worker call plus one checked verifier judgment), same reasoning ``adjudication_client`` already uses, not ``keyman_extraction``/``entity_relationship_classification``'s - single-pass ``mode="route"`` structured extraction. + single-pass ``mode="auto"`` structured extraction. """ available = True diff --git a/tests/test_adaptive_orchestrator_default.py b/tests/test_adaptive_orchestrator_default.py new file mode 100644 index 000000000..3de1dd412 --- /dev/null +++ b/tests/test_adaptive_orchestrator_default.py @@ -0,0 +1,49 @@ +"""LineageWeave delegates product-default LLM execution to auto policy.""" + +from __future__ import annotations + +from pathlib import Path + +from lineageweave import post_evaluation + + +def test_post_evaluation_adapter_defaults_to_auto(monkeypatch) -> None: + observed: dict[str, object] = {} + + def fake_post_json(url, payload, *, headers, timeout): + observed.update( + url=url, + payload=payload, + headers=headers, + timeout=timeout, + ) + return {"choices": [{"message": {"content": "{}"}}]} + + monkeypatch.setattr(post_evaluation, "post_json", fake_post_json) + adapter = post_evaluation._OrchestratorCompleteAdapter( + "https://orchestrator.example.test", "inference_token" + ) + adapter.complete([{"role": "user", "content": "Evaluate this evidence."}]) + + assert observed["payload"]["mode"] == "auto" + + +def test_post_evaluation_judge_uses_auto_by_default() -> None: + client = post_evaluation.ContextualOrchestratorPostEvaluationClient( + "https://orchestrator.example.test", "inference_token" + ) + assert client._judge.mode == "auto" + + +def test_runtime_clients_do_not_force_single_model_route() -> None: + package_root = Path(__file__).resolve().parents[1] / "lineageweave" + violations: list[str] = [] + for path in sorted(package_root.glob("*.py")): + text = path.read_text(encoding="utf-8") + if '"mode": "route"' in text or "'mode': 'route'" in text: + violations.append(f"{path.name}: request payload") + if 'mode="route"' in text or "mode='route'" in text: + violations.append(f"{path.name}: constructor/call default") + if 'mode: str = "route"' in text or "mode: str = 'route'" in text: + violations.append(f"{path.name}: typed default") + assert violations == [] diff --git a/tests/test_contextual_orchestrator_default_policy.py b/tests/test_contextual_orchestrator_default_policy.py new file mode 100644 index 000000000..6c5eb5d7c --- /dev/null +++ b/tests/test_contextual_orchestrator_default_policy.py @@ -0,0 +1,40 @@ +"""Contract tests for adaptive contextual-orchestrator consumer defaults.""" +from __future__ import annotations + +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] +ACTIVE_CLIENTS = ( + "lineageweave/post_summary.py", + "lineageweave/post_evaluation.py", + "lineageweave/keyman_extraction.py", + "lineageweave/commitment_extraction.py", + "lineageweave/post_chat.py", + "lineageweave/entity_relationship_classification.py", +) + + +class AdaptiveOrchestratorDefaultTest(unittest.TestCase): + """Protect production clients from regressing to forced one-model routing.""" + + def test_active_clients_use_auto_and_never_force_route(self) -> None: + for relative in ACTIVE_CLIENTS: + source = (ROOT / relative).read_text(encoding="utf-8") + with self.subTest(path=relative): + self.assertNotIn('"mode": "route"', source) + self.assertNotIn('mode="route"', source) + self.assertNotIn('mode: str = "route"', source) + self.assertTrue( + '"mode": "auto"' in source + or 'mode="auto"' in source + or 'mode: str = "auto"' in source + ) + + def test_high_stakes_adjudication_retains_explicit_checked_override(self) -> None: + source = (ROOT / "lineageweave/adjudication_client.py").read_text(encoding="utf-8") + self.assertIn('"mode": "verify"', source) + + +if __name__ == "__main__": + unittest.main() From a81544042bc824230c8b6de6c56c382c5982b11e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:17:54 +0000 Subject: [PATCH 2/4] test(ai): lock auto and verify orchestrator transport contracts Split auto and verify client lists so a post-chat docstring cannot satisfy the auto policy. Add wire-level verify assertions for citation chat and lineage adjudication, and name both exceptions in the Unreleased changelog. Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 14 ++-- CHANGELOG.md | 4 +- docs/lineage-bi-research-notes.md | 7 +- tests/test_adaptive_orchestrator_default.py | 64 +++++++++++++++++- ..._contextual_orchestrator_default_policy.py | 65 +++++++++++++++---- 5 files changed, 127 insertions(+), 27 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 617b8b95d..094a0aeec 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -83,17 +83,17 @@ flowchart LR | `server.py` | Stdlib HTTP server: `GET /api/lineage` (JSON graph) + static viewer | | `web/index.html` | Self-contained SVG DAG viewer, no build step, no external script dependency | -> **Known local-test-environment limitation:** `adjudication_client.py`'s -> `mode="verify"` call depends on contextual-orchestrator's -> `TaskOrchestrator.route_and_verify`, which as of this writing is still -> an open, unmerged upstream PR +> **Known local-test-environment limitation:** `adjudication_client.py` +> and `post_chat.py` send `mode="verify"` (ADR-0013). That call depends +> on contextual-orchestrator's `TaskOrchestrator.route_and_verify`, +> which as of this writing is still an open, unmerged upstream PR > (`ContextualWisdomLab/contextual-orchestrator#149`). Until it merges, -> the four adjudication/chat tests that exercise `mode="verify"` against +> the live adjudication/chat tests that exercise `mode="verify"` against > a real orchestrator fail with `invalid_mode` (the deployed `main` only > accepts `auto`/`route`/`conduct`) -- confirmed by reproducing the same > `400` directly against the orchestrator's own `/v1/chat/completions`, -> not caused by anything in this repo. `mode="route"` (every other -> pluggable client) is unaffected. +> not caused by anything in this repo. Ordinary product adapters request +> `mode="auto"` and are unaffected. ## Design decisions worth naming diff --git a/CHANGELOG.md b/CHANGELOG.md index 709e76880..d4b13c04d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,8 @@ All notable changes to this project are documented here. Format follows - Product LLM adapters now request contextual-orchestrator `mode="auto"` rather than forcing a one-model route. The orchestrator owns the quality-sufficient route, verification, or - conducted workflow; the explicit adjudication `verify` contract - remains unchanged. + conducted workflow. Citation-bearing post-chat and lineage + adjudication keep their explicit `verify` contracts. ## [0.71.0] - 2026-08-14 diff --git a/docs/lineage-bi-research-notes.md b/docs/lineage-bi-research-notes.md index acd55620b..07079230b 100644 --- a/docs/lineage-bi-research-notes.md +++ b/docs/lineage-bi-research-notes.md @@ -254,9 +254,10 @@ classified into the closed `{our_side, counterparty}` set is dropped rather than guessed. N:N organization attachments are slot-filling on that mention (a person may have zero, one, or several affiliations in the same post), not a second independent NER pass. The live client -calls contextual-orchestrator (`mode="route"`) rather than a raw LLM -API so reasoning-effort allocation stays centralized with the -adjudication channel. Proven for real during development against +calls contextual-orchestrator (`mode="auto"`) rather than a raw LLM +API so the orchestration plane can allocate route, verify, or a +deeper workflow; adjudication and post-chat keep explicit +`mode="verify"`. Proven for real during development against `fixtures.ambiguous_keyman_post` when orchestrator credentials are set; the default suite asserts the parser and the never-fake null client. diff --git a/tests/test_adaptive_orchestrator_default.py b/tests/test_adaptive_orchestrator_default.py index 3de1dd412..6b88197d3 100644 --- a/tests/test_adaptive_orchestrator_default.py +++ b/tests/test_adaptive_orchestrator_default.py @@ -4,7 +4,8 @@ from pathlib import Path -from lineageweave import post_evaluation +from lineageweave import adjudication_client, post_chat, post_evaluation +from lineageweave.post_chat import ChatSourceDocument, ContextualOrchestratorPostChatClient def test_post_evaluation_adapter_defaults_to_auto(monkeypatch) -> None: @@ -35,6 +36,67 @@ def test_post_evaluation_judge_uses_auto_by_default() -> None: assert client._judge.mode == "auto" +def test_post_chat_requests_verify_mode(monkeypatch) -> None: + """Citation chat must send verify on the wire, not a docstring mention of auto.""" + + observed: dict[str, object] = {} + + def fake_post_json(url, payload, *, headers, timeout): + observed["payload"] = payload + return { + "choices": [ + { + "message": { + "content": ( + '{"answer_text": "The follow-up names the same bid.",' + ' "cited_source_numbers": [1]}' + ) + } + } + ] + } + + monkeypatch.setattr(post_chat, "post_json", fake_post_json) + client = ContextualOrchestratorPostChatClient( + "https://orchestrator.example.test", "inference_token" + ) + answer = client.answer( + "What happened between these events?", + [ + ChatSourceDocument( + post_id="post-bid-follow-up", + post_title="Bid follow-up", + post_body="Northridge asked to confirm the bid date.", + ) + ], + ) + + assert answer.cited_post_ids == ("post-bid-follow-up",) + assert observed["payload"]["mode"] == "verify" + + +def test_adjudication_requests_verify_mode(monkeypatch) -> None: + """Lineage adjudication must send verify on the wire, not a source substring.""" + + observed: dict[str, object] = {} + + def fake_post_json(url, payload, *, headers, timeout): + observed["payload"] = payload + return {"choices": [{"message": {"content": "0.91"}}]} + + monkeypatch.setattr(adjudication_client, "post_json", fake_post_json) + client = adjudication_client.ContextualOrchestratorAdjudicationClient( + "https://orchestrator.example.test", "inference_token" + ) + confidence = client.judge( + "Quarterly budget review meeting notes", + "Budget review follow-up: revised quarterly numbers", + ) + + assert confidence == 0.91 + assert observed["payload"]["mode"] == "verify" + + def test_runtime_clients_do_not_force_single_model_route() -> None: package_root = Path(__file__).resolve().parents[1] / "lineageweave" violations: list[str] = [] diff --git a/tests/test_contextual_orchestrator_default_policy.py b/tests/test_contextual_orchestrator_default_policy.py index 6c5eb5d7c..24a023e1a 100644 --- a/tests/test_contextual_orchestrator_default_policy.py +++ b/tests/test_contextual_orchestrator_default_policy.py @@ -1,39 +1,76 @@ """Contract tests for adaptive contextual-orchestrator consumer defaults.""" + from __future__ import annotations from pathlib import Path import unittest ROOT = Path(__file__).resolve().parents[1] -ACTIVE_CLIENTS = ( +AUTO_CLIENTS = ( "lineageweave/post_summary.py", "lineageweave/post_evaluation.py", "lineageweave/keyman_extraction.py", "lineageweave/commitment_extraction.py", - "lineageweave/post_chat.py", "lineageweave/entity_relationship_classification.py", ) +VERIFY_CLIENTS = ( + "lineageweave/post_chat.py", + "lineageweave/adjudication_client.py", +) +_ROUTE_MARKERS = ( + '"mode": "route"', + 'mode="route"', + 'mode: str = "route"', +) +_AUTO_MARKERS = ( + '"mode": "auto"', + 'mode="auto"', + 'mode: str = "auto"', +) +_VERIFY_MARKERS = ( + '"mode": "verify"', + 'mode="verify"', + 'mode: str = "verify"', +) + + +def _source(relative: str) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def _contains_any(source: str, markers: tuple[str, ...]) -> bool: + return any(marker in source for marker in markers) class AdaptiveOrchestratorDefaultTest(unittest.TestCase): """Protect production clients from regressing to forced one-model routing.""" - def test_active_clients_use_auto_and_never_force_route(self) -> None: - for relative in ACTIVE_CLIENTS: - source = (ROOT / relative).read_text(encoding="utf-8") + def test_auto_clients_request_auto_and_never_force_route(self) -> None: + for relative in AUTO_CLIENTS: + source = _source(relative) with self.subTest(path=relative): - self.assertNotIn('"mode": "route"', source) - self.assertNotIn('mode="route"', source) - self.assertNotIn('mode: str = "route"', source) + for marker in _ROUTE_MARKERS: + self.assertNotIn(marker, source) self.assertTrue( - '"mode": "auto"' in source - or 'mode="auto"' in source - or 'mode: str = "auto"' in source + _contains_any(source, _AUTO_MARKERS), + f"{relative} must request mode=auto in executable source", ) - def test_high_stakes_adjudication_retains_explicit_checked_override(self) -> None: - source = (ROOT / "lineageweave/adjudication_client.py").read_text(encoding="utf-8") - self.assertIn('"mode": "verify"', source) + def test_verify_clients_keep_checked_judgment_and_never_force_route(self) -> None: + for relative in VERIFY_CLIENTS: + source = _source(relative) + with self.subTest(path=relative): + for marker in _ROUTE_MARKERS: + self.assertNotIn(marker, source) + self.assertTrue( + _contains_any(source, _VERIFY_MARKERS), + f"{relative} must request mode=verify in executable source", + ) + self.assertNotIn( + '"mode": "auto"', + source, + f"{relative} must send verify, not a payload-level auto default", + ) if __name__ == "__main__": From 32b6d8a9e7a6cf1fe4b96e828020457888c039e1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:29:58 +0000 Subject: [PATCH 3/4] test(ai): require payload literals for orchestrator mode scans A docstring mention of mode="auto" or mode="verify" can no longer satisfy the source-scan contract. Dict-payload clients must contain the JSON literal; post-evaluation keeps the typed default plus forwarded "mode": mode. Co-authored-by: Seongho Bae --- CHANGELOG.md | 4 +- ...daptive-contextual-orchestrator-default.md | 2 + ..._contextual_orchestrator_default_policy.py | 50 +++++++++++-------- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4b13c04d..077a32201 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,9 @@ All notable changes to this project are documented here. Format follows `mode="auto"` rather than forcing a one-model route. The orchestrator owns the quality-sufficient route, verification, or conducted workflow. Citation-bearing post-chat and lineage - adjudication keep their explicit `verify` contracts. + adjudication keep their explicit `verify` contracts. Source-scan + regressions require those payload literals so a docstring mention + cannot satisfy ADR-0013. ## [0.71.0] - 2026-08-14 diff --git a/docs/adr/0013-adaptive-contextual-orchestrator-default.md b/docs/adr/0013-adaptive-contextual-orchestrator-default.md index ee0402075..34113e005 100644 --- a/docs/adr/0013-adaptive-contextual-orchestrator-default.md +++ b/docs/adr/0013-adaptive-contextual-orchestrator-default.md @@ -21,6 +21,8 @@ LineageWeave continues to own strict output parsing, evidence identifiers, IRT p A structured task may still be served by one model when the adaptive policy determines that it is sufficient. Harder requests may receive a deeper workflow without changing the LineageWeave API. Consumers must retain returned orchestration and usage evidence when the gateway exposes it. +Contract tests require a payload-level `"mode": "auto"` or `"mode": "verify"` literal (or, for post-evaluation, `"mode": mode` plus `mode: str = "auto"`). A docstring mention of `mode="auto"` or `mode="verify"` is not sufficient. Wire tests call `answer()` / `judge()` / `evaluate()` and assert the outbound body. + ## References Omidvar, H., & Akhlaghi, V. (2026). *A communication-theoretic framework for LLM agents: Cost-aware adaptive reliability* [Preprint]. arXiv. https://doi.org/10.48550/arXiv.2605.09121 diff --git a/tests/test_contextual_orchestrator_default_policy.py b/tests/test_contextual_orchestrator_default_policy.py index 24a023e1a..3c292f6af 100644 --- a/tests/test_contextual_orchestrator_default_policy.py +++ b/tests/test_contextual_orchestrator_default_policy.py @@ -22,26 +22,16 @@ 'mode="route"', 'mode: str = "route"', ) -_AUTO_MARKERS = ( - '"mode": "auto"', - 'mode="auto"', - 'mode: str = "auto"', -) -_VERIFY_MARKERS = ( - '"mode": "verify"', - 'mode="verify"', - 'mode: str = "verify"', -) +_PAYLOAD_AUTO = '"mode": "auto"' +_PAYLOAD_VERIFY = '"mode": "verify"' +_TYPED_AUTO_DEFAULT = 'mode: str = "auto"' +_FORWARDED_MODE = '"mode": mode' def _source(relative: str) -> str: return (ROOT / relative).read_text(encoding="utf-8") -def _contains_any(source: str, markers: tuple[str, ...]) -> bool: - return any(marker in source for marker in markers) - - class AdaptiveOrchestratorDefaultTest(unittest.TestCase): """Protect production clients from regressing to forced one-model routing.""" @@ -51,9 +41,14 @@ def test_auto_clients_request_auto_and_never_force_route(self) -> None: with self.subTest(path=relative): for marker in _ROUTE_MARKERS: self.assertNotIn(marker, source) - self.assertTrue( - _contains_any(source, _AUTO_MARKERS), - f"{relative} must request mode=auto in executable source", + if relative.endswith("post_evaluation.py"): + self.assertIn(_FORWARDED_MODE, source) + self.assertIn(_TYPED_AUTO_DEFAULT, source) + continue + self.assertIn( + _PAYLOAD_AUTO, + source, + f"{relative} must send a payload-level mode=auto literal", ) def test_verify_clients_keep_checked_judgment_and_never_force_route(self) -> None: @@ -62,16 +57,29 @@ def test_verify_clients_keep_checked_judgment_and_never_force_route(self) -> Non with self.subTest(path=relative): for marker in _ROUTE_MARKERS: self.assertNotIn(marker, source) - self.assertTrue( - _contains_any(source, _VERIFY_MARKERS), - f"{relative} must request mode=verify in executable source", + self.assertIn( + _PAYLOAD_VERIFY, + source, + f"{relative} must send a payload-level mode=verify literal", ) self.assertNotIn( - '"mode": "auto"', + _PAYLOAD_AUTO, source, f"{relative} must send verify, not a payload-level auto default", ) + def test_docstring_mode_mention_is_not_a_payload_literal(self) -> None: + """A class docstring contrast must not satisfy the auto/verify scan.""" + + source = ( + '"""Calls the orchestrator with mode="auto", not mode="verify"."""\n' + "body = {\"messages\": [], \"mode\": \"route\"}\n" + ) + self.assertNotIn(_PAYLOAD_AUTO, source) + self.assertNotIn(_PAYLOAD_VERIFY, source) + self.assertIn('mode="auto"', source) + self.assertIn('mode="verify"', source) + if __name__ == "__main__": unittest.main() From b1f1db6ca6627d50e8e6a0b6946b144e3b6c5f63 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:49:46 +0000 Subject: [PATCH 4/4] test(ai): scan orchestrator mode literals via AST Close the remaining ADR-0013 hole where a docstring JSON fragment could satisfy a whole-file substring scan. The honesty unit now runs the same helpers against a prose-only fixture. Orchestrator vision sends mode=auto; generic OpenAI-compat vision still omits it. Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 4 +- CHANGELOG.md | 8 +- ...daptive-contextual-orchestrator-default.md | 4 +- lineageweave/image_content.py | 46 ++-- ..._contextual_orchestrator_default_policy.py | 213 ++++++++++++++---- tests/test_image_content.py | 48 ++++ 6 files changed, 259 insertions(+), 64 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 094a0aeec..90b41a19b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -93,7 +93,9 @@ flowchart LR > accepts `auto`/`route`/`conduct`) -- confirmed by reproducing the same > `400` directly against the orchestrator's own `/v1/chat/completions`, > not caused by anything in this repo. Ordinary product adapters request -> `mode="auto"` and are unaffected. +> `mode="auto"` and are unaffected. Vision built by +> `orchestrator_vision_client` also sends `mode="auto"`; a generic +> OpenAI-compatible vision client omits `mode`. ## Design decisions worth naming diff --git a/CHANGELOG.md b/CHANGELOG.md index 077a32201..ff955ea65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,11 @@ All notable changes to this project are documented here. Format follows `mode="auto"` rather than forcing a one-model route. The orchestrator owns the quality-sufficient route, verification, or conducted workflow. Citation-bearing post-chat and lineage - adjudication keep their explicit `verify` contracts. Source-scan - regressions require those payload literals so a docstring mention - cannot satisfy ADR-0013. + adjudication keep their explicit `verify` contracts. Vision built + by `orchestrator_vision_client` also sends `mode="auto"`; a generic + OpenAI-compatible vision client still omits `mode`. Source-scan + regressions walk the AST for those payload literals so a docstring + mention, including quoted JSON, cannot satisfy ADR-0013. ## [0.71.0] - 2026-08-14 diff --git a/docs/adr/0013-adaptive-contextual-orchestrator-default.md b/docs/adr/0013-adaptive-contextual-orchestrator-default.md index 34113e005..bcc8e4cf0 100644 --- a/docs/adr/0013-adaptive-contextual-orchestrator-default.md +++ b/docs/adr/0013-adaptive-contextual-orchestrator-default.md @@ -9,7 +9,7 @@ LineageWeave used fixed single-worker `route` mode for structured extraction, su ## Decision -Ordinary LineageWeave LLM consumers request `mode="auto"`. +Ordinary LineageWeave LLM consumers request `mode="auto"`. That includes structured extraction, summarization, commitment derivation, relationship classification, post evaluation, and vision OCR/captioning when the client is built by `orchestrator_vision_client`. A generic `OpenAiCompatibleVisionClient` omits `mode` so an OpenAI-compatible gateway that rejects unknown fields still works. The orchestration plane owns provider/model selection, test-time compute, workflow depth, verification, fallback, and known-price optimization. Quality sufficiency is the first constraint; cost is minimized among execution paths that satisfy it. Unpriced models are not treated as free. @@ -21,7 +21,7 @@ LineageWeave continues to own strict output parsing, evidence identifiers, IRT p A structured task may still be served by one model when the adaptive policy determines that it is sufficient. Harder requests may receive a deeper workflow without changing the LineageWeave API. Consumers must retain returned orchestration and usage evidence when the gateway exposes it. -Contract tests require a payload-level `"mode": "auto"` or `"mode": "verify"` literal (or, for post-evaluation, `"mode": mode` plus `mode: str = "auto"`). A docstring mention of `mode="auto"` or `mode="verify"` is not sufficient. Wire tests call `answer()` / `judge()` / `evaluate()` and assert the outbound body. +Contract tests walk the AST for a payload-level `"mode": "auto"` or `"mode": "verify"` literal (or, for post-evaluation, `"mode": mode` plus `mode: str = "auto"`; for vision, the orchestrator factory passes `mode="auto"` and `describe()` writes it onto the body). A docstring mention of `mode="auto"` or `mode="verify"`, including a quoted `{"mode": "auto"}` fragment, is not sufficient. Wire tests call `answer()` / `judge()` / `evaluate()` / `describe()` and assert the outbound body. ## References diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py index 2f6839fcd..5d3014458 100644 --- a/lineageweave/image_content.py +++ b/lineageweave/image_content.py @@ -173,6 +173,7 @@ def __init__( *, timeout: float = 60.0, allow_insecure_http: bool = False, + mode: str | None = None, ) -> None: parsed = urlparse(base_url) if parsed.scheme not in {"http", "https"}: @@ -192,25 +193,32 @@ def __init__( self._api_key = api_key self._model = model self._timeout = timeout + # None keeps generic OpenAI-compatible gateways from rejecting an + # unknown ``mode`` field. ``orchestrator_vision_client`` sets + # ``mode="auto"`` so ADR-0013 applies on the orchestrator path. + self._mode = mode def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: data_uri = f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('ascii')}" + payload: dict = { + "model": self._model, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": _RESPONSE_FORMAT}, + {"type": "image_url", "image_url": {"url": data_uri}}, + ], + } + ], + "max_tokens": 300, + "temperature": 0.0, + } + if self._mode is not None: + payload["mode"] = self._mode body = post_json( f"{self._base_url}/chat/completions", - { - "model": self._model, - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": _RESPONSE_FORMAT}, - {"type": "image_url", "image_url": {"url": data_uri}}, - ], - } - ], - "max_tokens": 300, - "temperature": 0.0, - }, + payload, headers={"authorization": f"Bearer {self._api_key}"}, timeout=self._timeout, ) @@ -223,10 +231,11 @@ def orchestrator_vision_client(base_url: str, api_key: str, model: str) -> Image Other clients POST ``{base_url}/v1/chat/completions``; :class:`OpenAiCompatibleVisionClient` POSTs ``{base_url}/chat/completions``, - so this appends ``/v1`` unless already present. An ``http://`` orchestrator - (local docker) is allowed because the other channels already talk to the - same URL. A construct-time error degrades to the unavailable null rather - than crashing the request that asked for a description. + so this appends ``/v1`` unless already present. The factory passes + ``mode="auto"`` (ADR-0013). An ``http://`` orchestrator (local docker) + is allowed because the other channels already talk to the same URL. + A construct-time error degrades to the unavailable null rather than + crashing the request that asked for a description. """ if not (base_url and api_key and model): return NullImageContentClient() @@ -240,6 +249,7 @@ def orchestrator_vision_client(base_url: str, api_key: str, model: str) -> Image api_key=api_key, model=model, allow_insecure_http=parsed.scheme == "http", + mode="auto", ) except ValueError: return NullImageContentClient() diff --git a/tests/test_contextual_orchestrator_default_policy.py b/tests/test_contextual_orchestrator_default_policy.py index 3c292f6af..fa6beff89 100644 --- a/tests/test_contextual_orchestrator_default_policy.py +++ b/tests/test_contextual_orchestrator_default_policy.py @@ -1,8 +1,17 @@ -"""Contract tests for adaptive contextual-orchestrator consumer defaults.""" +"""Contract tests for adaptive contextual-orchestrator consumer defaults. + +These scans walk the AST of each production client. A class docstring +that mentions ``mode="auto"`` or quotes ``{"mode": "auto"}`` is not a +payload. Beginners can read this file as: "the live request body must +name auto or verify in executable code, not in the comment that +describes the client." +""" from __future__ import annotations +import ast from pathlib import Path +import tempfile import unittest ROOT = Path(__file__).resolve().parents[1] @@ -12,26 +21,154 @@ "lineageweave/keyman_extraction.py", "lineageweave/commitment_extraction.py", "lineageweave/entity_relationship_classification.py", + "lineageweave/image_content.py", ) VERIFY_CLIENTS = ( "lineageweave/post_chat.py", "lineageweave/adjudication_client.py", ) -_ROUTE_MARKERS = ( - '"mode": "route"', - 'mode="route"', - 'mode: str = "route"', -) -_PAYLOAD_AUTO = '"mode": "auto"' -_PAYLOAD_VERIFY = '"mode": "verify"' -_TYPED_AUTO_DEFAULT = 'mode: str = "auto"' -_FORWARDED_MODE = '"mode": mode' +# Typed ``complete(..., mode="auto")`` forwards the name into the body. +_FORWARDED_AUTO_CLIENTS = frozenset({"lineageweave/post_evaluation.py"}) +# Generic OpenAI-compat vision omits ``mode`` so unknown-field gateways +# do not 400. The orchestrator factory must pass ``mode="auto"``. +_FACTORY_AUTO_CLIENTS = frozenset({"lineageweave/image_content.py"}) def _source(relative: str) -> str: + """Read one repository file as UTF-8 text.""" + return (ROOT / relative).read_text(encoding="utf-8") +def _dict_mode_values(source: str) -> list[ast.expr]: + """Return every ``{"mode": ...}`` value that appears in executable code. + + Comments and docstrings are invisible to the AST, so a prose mention + of ``mode="auto"`` or a docstring JSON fragment cannot appear here. + """ + + values: list[ast.expr] = [] + for node in ast.walk(ast.parse(source)): + if not isinstance(node, ast.Dict): + continue + for key, value in zip(node.keys, node.values, strict=False): + if value is None or key is None: + continue + if isinstance(key, ast.Constant) and key.value == "mode": + values.append(value) + return values + + +def _literal_modes(source: str) -> set[str]: + """Return string literals used as dict ``mode`` values in *source*.""" + + found: set[str] = set() + for value in _dict_mode_values(source): + if isinstance(value, ast.Constant) and isinstance(value.value, str): + found.add(value.value) + return found + + +def _forwards_mode_name(source: str) -> bool: + """True when a request dict sets ``"mode": mode`` (typed default).""" + + return any(isinstance(value, ast.Name) and value.id == "mode" for value in _dict_mode_values(source)) + + +def _is_auto_constant(node: ast.expr | None) -> bool: + """True when *node* is the string literal ``auto``.""" + + return isinstance(node, ast.Constant) and node.value == "auto" + + +def _typed_auto_default(source: str) -> bool: + """True when a parameter or annotated assignment defaults ``mode`` to auto.""" + + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.AnnAssign): + target = node.target + if isinstance(target, ast.Name) and target.id == "mode" and _is_auto_constant(node.value): + return True + continue + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + args = node.args + if args.defaults: + named = args.args[-len(args.defaults) :] + for arg, default in zip(named, args.defaults, strict=True): + if arg.arg == "mode" and _is_auto_constant(default): + return True + for arg, default in zip(args.kwonlyargs, args.kw_defaults, strict=True): + if arg.arg == "mode" and _is_auto_constant(default): + return True + return False + + +def _call_kwarg_mode_auto(source: str) -> bool: + """True when some call passes ``mode="auto"`` (factory / constructor).""" + + for node in ast.walk(ast.parse(source)): + if not isinstance(node, ast.Call): + continue + for keyword in node.keywords: + if keyword.arg != "mode": + continue + if isinstance(keyword.value, ast.Constant) and keyword.value.value == "auto": + return True + return False + + +def _assigns_payload_mode(source: str) -> bool: + """True when executable code writes ``payload["mode"] = ...``.""" + + for node in ast.walk(ast.parse(source)): + if not isinstance(node, ast.Assign): + continue + for target in node.targets: + if not isinstance(target, ast.Subscript): + continue + if isinstance(target.slice, ast.Constant) and target.slice.value == "mode": + return True + return False + + +def _assert_auto_client(source: str, relative: str) -> None: + """Fail unless *source* is an executable auto-mode consumer. + + Docstrings, comments, and quoted JSON fragments do not count. + """ + + modes = _literal_modes(source) + if "route" in modes: + raise AssertionError(f"{relative} must not send a payload-level mode=route literal") + if relative in _FORWARDED_AUTO_CLIENTS: + if not _forwards_mode_name(source): + raise AssertionError(f"{relative} must forward mode into the request body") + if not _typed_auto_default(source): + raise AssertionError(f"{relative} must default the forwarded mode to auto") + return + if relative in _FACTORY_AUTO_CLIENTS: + if not _call_kwarg_mode_auto(source): + raise AssertionError(f"{relative} must pass mode=auto from the orchestrator factory") + if not _assigns_payload_mode(source): + raise AssertionError(f"{relative} must write mode onto the outbound payload") + return + if "auto" not in modes: + raise AssertionError(f"{relative} must send a payload-level mode=auto literal") + + +def _assert_verify_client(source: str, relative: str) -> None: + """Fail unless *source* is an executable verify-mode consumer.""" + + modes = _literal_modes(source) + if "route" in modes: + raise AssertionError(f"{relative} must not send a payload-level mode=route literal") + if "auto" in modes: + raise AssertionError(f"{relative} must send verify, not a payload-level auto default") + if "verify" not in modes: + raise AssertionError(f"{relative} must send a payload-level mode=verify literal") + + class AdaptiveOrchestratorDefaultTest(unittest.TestCase): """Protect production clients from regressing to forced one-model routing.""" @@ -39,46 +176,42 @@ def test_auto_clients_request_auto_and_never_force_route(self) -> None: for relative in AUTO_CLIENTS: source = _source(relative) with self.subTest(path=relative): - for marker in _ROUTE_MARKERS: - self.assertNotIn(marker, source) - if relative.endswith("post_evaluation.py"): - self.assertIn(_FORWARDED_MODE, source) - self.assertIn(_TYPED_AUTO_DEFAULT, source) - continue - self.assertIn( - _PAYLOAD_AUTO, - source, - f"{relative} must send a payload-level mode=auto literal", - ) + _assert_auto_client(source, relative) def test_verify_clients_keep_checked_judgment_and_never_force_route(self) -> None: for relative in VERIFY_CLIENTS: source = _source(relative) with self.subTest(path=relative): - for marker in _ROUTE_MARKERS: - self.assertNotIn(marker, source) - self.assertIn( - _PAYLOAD_VERIFY, - source, - f"{relative} must send a payload-level mode=verify literal", - ) - self.assertNotIn( - _PAYLOAD_AUTO, - source, - f"{relative} must send verify, not a payload-level auto default", - ) + _assert_verify_client(source, relative) def test_docstring_mode_mention_is_not_a_payload_literal(self) -> None: - """A class docstring contrast must not satisfy the auto/verify scan.""" + """The same helpers used on production clients must reject prose-only mode.""" source = ( - '"""Calls the orchestrator with mode="auto", not mode="verify"."""\n' - "body = {\"messages\": [], \"mode\": \"route\"}\n" + '"""Calls the orchestrator with mode="auto", not mode="verify".\n' + 'Also quotes {"mode": "auto"} and {"mode": "verify"} as prose.\n' + '"""\n' + 'body = {"messages": []}\n' + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "docstring_only_client.py" + path.write_text(source, encoding="utf-8") + loaded = path.read_text(encoding="utf-8") + with self.assertRaises(AssertionError): + _assert_auto_client(loaded, "lineageweave/docstring_only_client.py") + with self.assertRaises(AssertionError): + _assert_verify_client(loaded, "lineageweave/docstring_only_client.py") + self.assertEqual(_literal_modes(loaded), set()) + self.assertIn('mode="auto"', loaded) + self.assertIn('"mode": "auto"', loaded) + + def test_scan_accepts_an_executable_auto_payload(self) -> None: + """A real request dict is what the auto scan is looking for.""" + + _assert_auto_client( + 'body = {"messages": [], "mode": "auto"}\n', + "lineageweave/real_auto_client.py", ) - self.assertNotIn(_PAYLOAD_AUTO, source) - self.assertNotIn(_PAYLOAD_VERIFY, source) - self.assertIn('mode="auto"', source) - self.assertIn('mode="verify"', source) if __name__ == "__main__": diff --git a/tests/test_image_content.py b/tests/test_image_content.py index 572909d6f..fb2cb0d55 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -98,6 +98,7 @@ def test_vision_client_accepts_https_urls_by_default() -> None: model="unused", ) assert https_client._base_url == "https://gateway.example/v1" + assert https_client._mode is None def test_vision_client_rejects_plain_http_by_default() -> None: @@ -126,6 +127,7 @@ def test_orchestrator_vision_client_appends_v1_and_allows_local_http() -> None: client = orchestrator_vision_client("http://127.0.0.1:8000", "key", "vision-model") assert isinstance(client, OpenAiCompatibleVisionClient) assert client._base_url == "http://127.0.0.1:8000/v1" + assert client._mode == "auto" def test_orchestrator_vision_client_does_not_double_v1() -> None: @@ -140,6 +142,52 @@ def test_orchestrator_vision_client_is_null_when_unconfigured() -> None: assert client.available is False +def test_orchestrator_vision_client_requests_auto_mode(monkeypatch) -> None: + """Orchestrator-built vision must send mode=auto on the wire, not a docstring.""" + + observed: dict[str, object] = {} + + def fake_post_json(url, payload, *, headers, timeout): + observed["payload"] = payload + return { + "choices": [ + {"message": {"content": "TEXT: NONE\nCAPTION: A 1x1 pixel.\nTAGS: pixel"}} + ] + } + + monkeypatch.setattr("lineageweave.image_content.post_json", fake_post_json) + client = orchestrator_vision_client("https://orchestrator.example.test", "key", "vision-model") + assert isinstance(client, OpenAiCompatibleVisionClient) + description = client.describe(base64.b64decode(_TINY_PNG_B64), "image/png") + + assert observed["payload"]["mode"] == "auto" + assert description.caption == "A 1x1 pixel." + + +def test_generic_vision_client_omits_mode_for_openai_compat(monkeypatch) -> None: + """A raw OpenAI-compatible gateway must not receive an unknown mode field.""" + + observed: dict[str, object] = {} + + def fake_post_json(url, payload, *, headers, timeout): + observed["payload"] = payload + return { + "choices": [ + {"message": {"content": "TEXT: NONE\nCAPTION: A 1x1 pixel.\nTAGS: pixel"}} + ] + } + + monkeypatch.setattr("lineageweave.image_content.post_json", fake_post_json) + client = OpenAiCompatibleVisionClient( + base_url="https://gateway.example/v1", + api_key="key", + model="vision-model", + ) + client.describe(base64.b64decode(_TINY_PNG_B64), "image/png") + + assert "mode" not in observed["payload"] + + def test_image_content_client_protocol_stub_raises() -> None: """The Protocol method is a real stub, not a no-op ellipsis, so a mistaken call cannot be mistaken for a successful empty description.