From 9848bf09ac946305fb9d4d524c526ce5dd548e08 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Wed, 16 Sep 2026 18:15:32 -0400 Subject: [PATCH 01/52] fix(pixel): separate model promotion and route completion receipts --- ods/bin/pixel_access_bridge.py | 17 +++++++++-- ods/bin/pixel_model_coordinator.py | 17 +++++++++-- .../tests/test_model_coordinator.py | 28 +++++++++++++++++++ ods/lib/pixel-uninstall.sh | 2 ++ ods/tests/test-pixel-uninstall.sh | 3 ++ ods/tests/test_pixel_model_transition.py | 25 ++++++++++++++++- 6 files changed, 87 insertions(+), 5 deletions(-) diff --git a/ods/bin/pixel_access_bridge.py b/ods/bin/pixel_access_bridge.py index e7a4aabcdf..b34445b687 100644 --- a/ods/bin/pixel_access_bridge.py +++ b/ods/bin/pixel_access_bridge.py @@ -999,9 +999,22 @@ def remove_model_journal(self): finally: os.close(directory) def model_completion(self, request=None): - path = self.state / "model-completed.json" + path = self.state / "model-promotion-completed.json" + legacy = False + if not path.exists(): + path = self.state / "model-completed.json" + legacy = True if not path.exists(): return None value = private_json(path, 0, 4096) + # Older releases shared this filename with browser model switching. + # A valid receipt from that separate transaction is not a promotion + # completion; malformed state still fails closed. + if legacy and type(value) is dict and set(value) == {"transactionId", "outcome", "configSha256"}: + if (type(value["transactionId"]) is not str or not HEX.fullmatch(value["transactionId"]) + or value["outcome"] not in ("commit", "rollback") + or type(value["configSha256"]) is not str or not HEX.fullmatch(value["configSha256"])): + raise AccessError("model-recovery-required") + return None if (type(value) is not dict or set(value) != {"kind", "transaction_id", "outcome", "config_sha256"} or value.get("kind") != "model-completion" @@ -1181,7 +1194,7 @@ def model_finish(self, request): or released_edge.get("streams")): raise try: - atomic_json(self.state / "model-completed.json", { + atomic_json(self.state / "model-promotion-completed.json", { "kind": "model-completion", "transaction_id": pending["transaction_id"], "outcome": request["outcome"], "config_sha256": config["config_sha256"]}) except OSError: diff --git a/ods/bin/pixel_model_coordinator.py b/ods/bin/pixel_model_coordinator.py index 2cd6cb9fd5..7b9c347c50 100644 --- a/ods/bin/pixel_model_coordinator.py +++ b/ods/bin/pixel_model_coordinator.py @@ -142,8 +142,21 @@ def _status(bridge): raise AccessError("model-runtime-mismatch") if (_config(bridge)[1] != config_sha or _identity(bridge) != identity or native.get("revision") != observed["revision"]): raise AccessError("model-inspection-changed") - done_path = bridge.state / "model-completed.json" + done_path = bridge.state / "model-route-completed.json" + legacy = False + if not done_path.exists(): + done_path = bridge.state / "model-completed.json" + legacy = True done = private_json(done_path, 0, 8192) if done_path.exists() else None + # The legacy filename was also used by install-time model promotion. + # Accept its valid receipt as belonging to that other transaction, never + # as evidence that a browser model switch completed. + if legacy and type(done) is dict and set(done) == {"kind", "transaction_id", "outcome", "config_sha256"}: + if (done["kind"] != "model-completion" or not checksum(done["transaction_id"]) + or done["outcome"] not in ("applied", "rolled-back") + or not checksum(done["config_sha256"])): + raise AccessError("invalid-model-completion") + done = None if done and (not checksum(done.get("transactionId")) or done.get("outcome") not in ("commit", "rollback") or not checksum(done.get("configSha256"))): raise AccessError("invalid-model-completion") done = done if done and done["configSha256"] == config_sha else None @@ -234,7 +247,7 @@ def control(bridge, operation, request=None): _verify(bridge, journal, expected_sha) if owner["pending"]: worker("model-finish", model_outcome=outcome) journal.update(phase="releasing", outcome=outcome); _write(bridge, journal) - atomic_json(bridge.state / "model-completed.json", {"transactionId": journal["transactionId"], "outcome": outcome, "configSha256": expected_sha}) + atomic_json(bridge.state / "model-route-completed.json", {"transactionId": journal["transactionId"], "outcome": outcome, "configSha256": expected_sha}) bridge.edge("release", journal["token"], journal["edge_revision"]) bridge.native("release", journal["token"]) try: diff --git a/ods/extensions/services/pixel-agent/tests/test_model_coordinator.py b/ods/extensions/services/pixel-agent/tests/test_model_coordinator.py index 9dfedeb188..5a7b0f80b9 100644 --- a/ods/extensions/services/pixel-agent/tests/test_model_coordinator.py +++ b/ods/extensions/services/pixel-agent/tests/test_model_coordinator.py @@ -78,6 +78,34 @@ def test_64k_to_16k_holds_through_actual_runtime_readback(adapter): assert finish(adapter)==done assert 'PRIVATE' not in json.dumps(done) +def test_bootstrap_completion_does_not_block_browser_model_switch(adapter): + # Fresh installs leave the old shared receipt in this exact format. + adapter.state.mkdir(mode=0o700) + promotion = {"kind": "model-completion", "transaction_id": ID, + "outcome": "applied", "config_sha256": sha(adapter.path)} + atomic_json(adapter.state / "model-completed.json", promotion) + assert c.control(adapter, 'model-status')['status'] == 'ready' + begin(adapter);apply(adapter) + done=finish(adapter) + assert done['status'] == 'completed' and done['outcome'] == 'commit' + assert (adapter.state / "model-route-completed.json").exists() + assert json.loads((adapter.state / "model-completed.json").read_text()) == promotion + +def test_legacy_browser_completion_is_read_without_rewriting_it(adapter): + adapter.state.mkdir(mode=0o700) + receipt={"transactionId": ID, "outcome": "commit", "configSha256": sha(adapter.path)} + atomic_json(adapter.state / "model-completed.json", receipt) + status=c.control(adapter, 'model-status') + assert status['status'] == 'completed' and status['transactionId'] == ID + assert not (adapter.state / "model-route-completed.json").exists() + +def test_malformed_legacy_promotion_receipt_fails_closed(adapter): + adapter.state.mkdir(mode=0o700) + atomic_json(adapter.state / "model-completed.json", {"kind": "model-completion", + "transaction_id": "bad", "outcome": "applied", "config_sha256": sha(adapter.path)}) + with pytest.raises(AccessError,match='invalid-model-completion'): + c.control(adapter,'model-status') + @pytest.mark.parametrize('failure',['preinvoke','lost-reply']) def test_lost_reply_or_partial_begin_can_restore_exact_bytes(adapter,failure): before=adapter.path.read_bytes();adapter.failure=failure diff --git a/ods/lib/pixel-uninstall.sh b/ods/lib/pixel-uninstall.sh index 8085ecc85e..e121a72c4b 100644 --- a/ods/lib/pixel-uninstall.sh +++ b/ods/lib/pixel-uninstall.sh @@ -233,6 +233,8 @@ state_limits = { "service-baseline.json": 64 * 1024, "model-before.json": 8 * 1024 * 1024, "model-completed.json": 256 * 1024, + "model-promotion-completed.json": 256 * 1024, + "model-route-completed.json": 256 * 1024, "settings-verified.json": 256 * 1024, "provider-root-plan.json": 8 * 1024 * 1024, "provider-root-managed.json": 8 * 1024 * 1024, diff --git a/ods/tests/test-pixel-uninstall.sh b/ods/tests/test-pixel-uninstall.sh index ef7818fcd7..309c4ce5eb 100755 --- a/ods/tests/test-pixel-uninstall.sh +++ b/ods/tests/test-pixel-uninstall.sh @@ -1833,6 +1833,9 @@ else fi write_access_fixture +printf '{}\n' > "$ACCESS_STATE/model-promotion-completed.json" +printf '{}\n' > "$ACCESS_STATE/model-route-completed.json" +chmod 0600 "$ACCESS_STATE/model-promotion-completed.json" "$ACCESS_STATE/model-route-completed.json" if ods_pixel_uninstall_managed "$INSTALL_DIR" "$HOME_DIR"; then [[ ! -e "$ACCESS_STATE" && ! -e "$LIBEXEC_DIR/ods-pixel-access" \ && ! -e "$ETC_DIR/pixel-access.json" && ! -e "$SYSTEMD_DIR/ods-pixel-access.service" ]] \ diff --git a/ods/tests/test_pixel_model_transition.py b/ods/tests/test_pixel_model_transition.py index 827c050885..d83c691f77 100644 --- a/ods/tests/test_pixel_model_transition.py +++ b/ods/tests/test_pixel_model_transition.py @@ -255,11 +255,34 @@ def test_finish_reacquires_reproofs_and_releases(self): result = bridge.model_finish({"transaction_id": HEX_A, "outcome": "applied"}) self.assertEqual(result, {"status": "released", "outcome": "applied"}) self.assertFalse((bridge.state / "transition.json").exists()) - self.assertTrue((bridge.state / "model-completed.json").exists()) + self.assertTrue((bridge.state / "model-promotion-completed.json").exists()) self.assertIn("discover-installing", bridge.calls) self.assertLess(bridge.calls.index("verify:sandboxed"), bridge.calls.index("native:release")) self.assertLess(bridge.calls.index("native:release"), bridge.calls.index("edge:release")) + def test_promotion_completion_ignores_legacy_browser_receipt(self): + with tempfile.TemporaryDirectory() as root: + bridge = FakeBridge(root) + legacy = bridge.state / "model-completed.json" + legacy.write_text(json.dumps({"transactionId": HEX_A, "outcome": "commit", + "configSha256": HEX_B}), encoding="utf-8") + self.assertIsNone(bridge.model_completion()) + legacy.write_text(json.dumps({"transactionId": "bad", "outcome": "commit", + "configSha256": HEX_B}), encoding="utf-8") + with self.assertRaisesRegex(AccessError, "model-recovery-required"): + bridge.model_completion() + + def test_promotion_completion_reads_legacy_and_prefers_namespaced_receipt(self): + with tempfile.TemporaryDirectory() as root: + bridge = FakeBridge(root) + legacy = {"kind": "model-completion", "transaction_id": HEX_A, + "outcome": "applied", "config_sha256": HEX_B} + (bridge.state / "model-completed.json").write_text(json.dumps(legacy), encoding="utf-8") + self.assertEqual(bridge.model_completion(), legacy) + current = {**legacy, "transaction_id": HEX_C} + (bridge.state / "model-promotion-completed.json").write_text(json.dumps(current), encoding="utf-8") + self.assertEqual(bridge.model_completion(), current) + def test_finish_replays_exact_root_completion_after_lost_socket_reply(self): with tempfile.TemporaryDirectory() as root: bridge = FakeBridge(root) From afe74085e572c167f44563efc68476f1a910e179 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 16 Sep 2026 20:05:29 -0400 Subject: [PATCH 02/52] fix(pixel): keep managed marker bound to model switches --- ods/bin/pixel_model_coordinator.py | 83 ++++++++++++- .../tests/test_model_coordinator.py | 110 ++++++++++++++++++ 2 files changed, 191 insertions(+), 2 deletions(-) diff --git a/ods/bin/pixel_model_coordinator.py b/ods/bin/pixel_model_coordinator.py index 7b9c347c50..2f1c16f65d 100644 --- a/ods/bin/pixel_model_coordinator.py +++ b/ods/bin/pixel_model_coordinator.py @@ -2,6 +2,8 @@ import hashlib import json import os +import stat +import tempfile import time from pixel_access_bridge import AccessError, UNIT, atomic_json, private_json, digest, remaining from pixel_settings.coordinator import _read, _identity, _valid_identity @@ -14,7 +16,7 @@ def _sha(value): def _journal(value): required = {"kind", "phase", "token", "transactionId", "edge_revision", "edgeHeld", "beforeSha", "afterSha", "target", "boundary", "mode", "beforeIdentity"} - if (type(value) is not dict or set(value) - required - {"outcome"} or not required <= set(value) + if (type(value) is not dict or set(value) - required - {"outcome", "markerBeforeSha"} or not required <= set(value) or value["kind"] != "model" or value["phase"] not in ("acquiring", "held", "applying", "applied", "restoring", "releasing") or any(not checksum(value[key]) for key in ("token", "transactionId", "edge_revision", "beforeSha")) or value["afterSha"] is not None and not checksum(value["afterSha"]) @@ -24,6 +26,8 @@ def _journal(value): or type(value["edgeHeld"]) is not bool or "outcome" in value and value["outcome"] not in ("commit", "rollback")): raise AccessError("invalid-model-transition") + if "markerBeforeSha" in value and not checksum(value["markerBeforeSha"]): + raise AccessError("invalid-model-transition") if value["target"] is not None: target(value["target"]) return value @@ -36,6 +40,76 @@ def _config(bridge): return _read(bridge.home / ".openclaw/openclaw.json", bridge.owner.pw_uid) +def _marker_digest(config): + if type(config) is not dict: + raise AccessError("model-marker-invalid") + canonical = json.dumps(config, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(b"ods-pixel-openclaw-v1\0" + canonical).hexdigest() + + +def _managed_marker(bridge): + path = bridge.home / ".config/ods/pixel-managed.json" + for directory, unsafe_bits in ((path.parent.parent, 0o022), (path.parent, 0o077)): + info = directory.lstat() + if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) + or info.st_uid != bridge.owner.pw_uid or info.st_mode & unsafe_bits): + raise AccessError("model-marker-unsafe") + marker = private_json(path, bridge.owner.pw_uid, 65536) + if (type(marker) is not dict or marker.get("schema_version") != 2 + or marker.get("manager") != "ods" or marker.get("state") != "ready" + or marker.get("initial_active_state") != "absent" + or marker.get("install_dir") != str(bridge.install) + or type(marker.get("configuration_sha256")) is not str + or not checksum(marker["configuration_sha256"])): + raise AccessError("model-marker-invalid") + return path, marker + + +def _bind_managed_marker(bridge, journal, expected_sha): + before = private_json(bridge.state / "model-before.json", 0, 8 * 1024 * 1024) + prior = _marker_digest(before) + # Pre-upgrade journals did not carry markerBeforeSha. Their root-owned + # model-before snapshot and the still-bound owner marker can prove the + # same prior configuration without silently adopting unrelated drift. + if prior != journal.get("markerBeforeSha", prior): + raise AccessError("model-before-changed") + config, config_sha = _config(bridge) + if config_sha != expected_sha: + raise AccessError("model-config-changed") + path, marker = _managed_marker(bridge) + current = _marker_digest(config) + if marker["configuration_sha256"] == current: + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: os.fsync(directory) + finally: os.close(directory) + return # A retry after the marker rename is idempotent. + if marker["configuration_sha256"] != prior: + raise AccessError("model-marker-drifted") + original = path.lstat() + marker["configuration_sha256"] = current + fd, temporary = tempfile.mkstemp(prefix=".pixel-managed.", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: + if os.geteuid() != bridge.owner.pw_uid or os.getegid() != bridge.owner.pw_gid: + os.fchown(handle.fileno(), bridge.owner.pw_uid, bridge.owner.pw_gid) + os.fchmod(handle.fileno(), 0o600) + json.dump(marker, handle, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + observed = path.lstat() + if ((observed.st_dev, observed.st_ino, observed.st_mtime_ns, observed.st_size) + != (original.st_dev, original.st_ino, original.st_mtime_ns, original.st_size)): + raise AccessError("model-marker-drifted") + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: os.fsync(directory) + finally: os.close(directory) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + def _readback(bridge, journal=None): native = bridge.native() if native.get("stopped"): raise AccessError("model-runtime-unavailable") @@ -193,9 +267,13 @@ def control(bridge, operation, request=None): if access["busy"]: raise AccessError("runtime-busy") if access["configured_mode"] not in ("sandboxed", "full-access"): raise AccessError("model-access-mode-unknown") config, config_sha = _config(bridge) + _, marker = _managed_marker(bridge) + if marker["configuration_sha256"] != _marker_digest(config): + raise AccessError("model-marker-drifted") journal = dict(kind="model", phase="acquiring", token=os.urandom(32).hex(), transactionId=request["transactionId"], edge_revision=access["_edge"]["revision"], edgeHeld=False, beforeSha=config_sha, afterSha=None, target=None, - boundary=bridge.unit_boundary(), mode=access["configured_mode"], beforeIdentity=_identity(bridge)) + boundary=bridge.unit_boundary(), mode=access["configured_mode"], beforeIdentity=_identity(bridge), + markerBeforeSha=marker["configuration_sha256"]) atomic_json(bridge.state / "model-before.json", config) _write(bridge, journal) _hold(bridge, journal) @@ -246,6 +324,7 @@ def control(bridge, operation, request=None): "config_sha256": expected_sha, "boundary": journal["boundary"]}) _verify(bridge, journal, expected_sha) if owner["pending"]: worker("model-finish", model_outcome=outcome) + _bind_managed_marker(bridge, journal, expected_sha) journal.update(phase="releasing", outcome=outcome); _write(bridge, journal) atomic_json(bridge.state / "model-route-completed.json", {"transactionId": journal["transactionId"], "outcome": outcome, "configSha256": expected_sha}) bridge.edge("release", journal["token"], journal["edge_revision"]) diff --git a/ods/extensions/services/pixel-agent/tests/test_model_coordinator.py b/ods/extensions/services/pixel-agent/tests/test_model_coordinator.py index 5a7b0f80b9..08a549a15f 100644 --- a/ods/extensions/services/pixel-agent/tests/test_model_coordinator.py +++ b/ods/extensions/services/pixel-agent/tests/test_model_coordinator.py @@ -4,6 +4,7 @@ from unittest import SkipTest raise SkipTest("POSIX owner coordination runs under Linux/WSL") import json +import hashlib import os from pathlib import Path import pytest @@ -24,6 +25,14 @@ def __init__(self, root): self.path=self.home / '.openclaw/openclaw.json' self.path.parent.mkdir(mode=0o700) atomic_json(self.path,config()) + config_directory = self.home / '.config' + config_directory.mkdir(mode=0o700) + marker_directory = config_directory / 'ods' + marker_directory.mkdir(mode=0o700) + self.marker_path = marker_directory / 'pixel-managed.json' + atomic_json(self.marker_path, {'schema_version': 2, 'manager': 'ods', 'state': 'ready', + 'initial_active_state': 'absent', + 'install_dir': str(self.install), 'configuration_sha256': c._marker_digest(config())}) self.loaded=json.loads(self.path.read_text()) self.owner_state=self.path.parent / '.ods-access-mode' def command(self,args,timeout=20): @@ -69,11 +78,16 @@ def apply(a):return c.control(a,'model-apply',{'transactionId':ID,'target':NEW}) def finish(a,outcome='commit'):return c.control(a,'model-finish',{'transactionId':ID,'outcome':outcome}) def test_64k_to_16k_holds_through_actual_runtime_readback(adapter): + previous_marker = json.loads(adapter.marker_path.read_text())['configuration_sha256'] assert begin(adapter)['pending'];assert adapter.native_phase==adapter.edge_phase=='held' result=apply(adapter);assert result['status']=='applied' and result['contract']==NEW assert adapter.log.count('restart')==1 assert adapter.native_phase==adapter.edge_phase=='held' done=finish(adapter);assert done['outcome']=='commit' and not done['pending'] + assert json.loads(adapter.marker_path.read_text())['configuration_sha256'] == c._marker_digest(json.loads(adapter.path.read_text())) + assert json.loads(adapter.marker_path.read_text())['configuration_sha256'] != previous_marker + assert adapter.marker_path.stat().st_mode & 0o777 == 0o600 + assert adapter.marker_path.stat().st_uid == os.getuid() assert adapter.native_phase==adapter.edge_phase=='idle' assert finish(adapter)==done assert 'PRIVATE' not in json.dumps(done) @@ -109,6 +123,7 @@ def test_malformed_legacy_promotion_receipt_fails_closed(adapter): @pytest.mark.parametrize('failure',['preinvoke','lost-reply']) def test_lost_reply_or_partial_begin_can_restore_exact_bytes(adapter,failure): before=adapter.path.read_bytes();adapter.failure=failure + prior_marker = adapter.marker_path.read_bytes() with pytest.raises(AccessError): begin(adapter) apply(adapter) @@ -116,8 +131,103 @@ def test_lost_reply_or_partial_begin_can_restore_exact_bytes(adapter,failure): adapter.failure=None restored=finish(adapter,'rollback') assert restored['outcome']=='rollback' and adapter.path.read_bytes()==before + assert adapter.marker_path.read_bytes() == prior_marker assert adapter.native_phase==adapter.edge_phase=='idle' + +def test_model_begin_refuses_unattested_marker_drift(adapter): + marker = json.loads(adapter.marker_path.read_text()) + marker['configuration_sha256'] = 'f' * 64 + atomic_json(adapter.marker_path, marker) + with pytest.raises(AccessError, match='model-marker-drifted'): + begin(adapter) + assert not adapter.pending() + + +@pytest.mark.parametrize('field,value', [ + ('manager', 'other'), ('state', 'deactivating'), + ('initial_active_state', 'present'), ('install_dir', '/other/install'), +]) +def test_model_begin_refuses_foreign_or_inactive_marker(adapter, field, value): + marker = json.loads(adapter.marker_path.read_text()) + marker[field] = value + atomic_json(adapter.marker_path, marker) + with pytest.raises(AccessError, match='model-marker-invalid'): + begin(adapter) + assert not adapter.pending() + + +def test_model_begin_refuses_writable_marker_directory(adapter): + adapter.marker_path.parent.chmod(0o777) + with pytest.raises(AccessError, match='model-marker-unsafe'): + begin(adapter) + assert not adapter.pending() + + +def test_model_begin_refuses_symlinked_marker(adapter): + alternate = adapter.marker_path.with_name('other.json') + adapter.marker_path.rename(alternate) + adapter.marker_path.symlink_to(alternate) + with pytest.raises((AccessError, OSError)): + begin(adapter) + assert not adapter.pending() + + +def test_marker_digest_matches_independent_installer_contract(): + canonical = json.dumps(config(), sort_keys=True, separators=(',', ':')).encode() + expected = hashlib.sha256(b'ods-pixel-openclaw-v1\0' + canonical).hexdigest() + assert c._marker_digest(config()) == expected + + +def test_model_finish_preserves_hold_if_marker_changes_during_switch(adapter): + original = json.loads(adapter.marker_path.read_text()) + begin(adapter); apply(adapter) + changed = dict(original, configuration_sha256='f' * 64) + atomic_json(adapter.marker_path, changed) + with pytest.raises(AccessError, match='model-marker-drifted'): + finish(adapter) + assert adapter.pending() and adapter.native_phase == adapter.edge_phase == 'held' + assert json.loads(adapter.marker_path.read_text()) == changed + atomic_json(adapter.marker_path, original) + assert finish(adapter)['outcome'] == 'commit' + + +def test_marker_rebind_is_idempotent_after_interrupted_release(adapter): + begin(adapter); apply(adapter) + adapter.fail = 'release' + with pytest.raises(AccessError): finish(adapter) + bound = adapter.marker_path.read_bytes() + adapter.fail = None + assert finish(adapter)['outcome'] == 'commit' + assert adapter.marker_path.read_bytes() == bound + + +def test_prepatch_pending_journal_can_complete_without_adopting_drift(adapter): + begin(adapter); apply(adapter) + journal = adapter.pending() + journal.pop('markerBeforeSha') + atomic_json(adapter.state / 'transition.json', journal) + assert finish(adapter)['outcome'] == 'commit' + assert json.loads(adapter.marker_path.read_text())['configuration_sha256'] == c._marker_digest(json.loads(adapter.path.read_text())) + + +def test_six_switches_keep_installer_marker_bound(adapter): + # Repeated browser cycles must not strand the uninstall/reinstall guard. + for cycle in range(1, 7): + transaction_id = f'{cycle:x}' * 64 + revision = c.control(adapter, 'model-status')['revision'] + c.control(adapter, 'model-begin', {'revision': revision, 'transactionId': transaction_id}) + c.control(adapter, 'model-apply', {'transactionId': transaction_id, 'target': NEW}) + c.control(adapter, 'model-finish', {'transactionId': transaction_id, 'outcome': 'commit'}) + marker = json.loads(adapter.marker_path.read_text()) + assert marker['configuration_sha256'] == c._marker_digest(json.loads(adapter.path.read_text())) + + +def test_normal_nonwritable_config_parent_is_accepted(adapter): + (adapter.home / '.config').chmod(0o755) + assert begin(adapter)['pending'] + assert finish(adapter, 'rollback')['outcome'] == 'rollback' + def test_partial_release_reacquires_owned_hold_and_finishes(adapter): begin(adapter);apply(adapter);adapter.fail='release' with pytest.raises(AccessError):finish(adapter) From f608b0355f0d7ad4def2a8c8c2e411299b54b3ec Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 16 Sep 2026 21:45:50 -0400 Subject: [PATCH 03/52] docs(models): scope Granite 3.2 Perplexica warning to Tower1 --- ods/config/model-library.json | 12 ++++++------ .../dashboard-api/tests/test_performance_oracle.py | 8 +++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ods/config/model-library.json b/ods/config/model-library.json index d60c5a8334..447a371fbd 100644 --- a/ods/config/model-library.json +++ b/ods/config/model-library.json @@ -779,12 +779,12 @@ "perplexica": { "status": "unsupported_until_revalidated", "label": "Perplexica revalidation required", - "reason": "Earlier fleet runs found Perplexica nonce failures with this model on Tower2 and M5. A 2026-09-15 Tower3 public-beta run again returned an apologetic Perplexica response without the verification phrase, while ODS Talk, LiteLLM, Open WebUI, and Pixel routed and answered on the selected Granite model. Perplexica answered the same harmless payload after Tower3 recovered its Qwen3.5-27B model. Keep Granite 3.2 2B out of all-app release coverage on these hosts until Perplexica passes a real revalidation.", - "evidence": "fleet-test/runs/2026-07-23T21-41-59Z-release-product-cf6dffb8e8bb-harness-47eedd6c0b40-hosts-six-scope-6cycle-switchboard/model-ui/cycle-005/tower2; fleet-test/runs/2026-07-23T21-41-59Z-release-product-cf6dffb8e8bb-harness-47eedd6c0b40-hosts-six-scope-6cycle-switchboard/model-ui/cycle-005/m5-mbp; ods-public-beta-fleet-green-20260911/release-four-host-2f985137-abb61b2-r46/model-ui/cycle-006/tower3/model-ui.json; Tower3 Perplexica live comparator 2026-09-15T16:59Z", - "recordedAt": "2026-09-15T16:59:00Z", - "productSha": "2f985137b9680151395d59d0498a7fa7e169ff2e", - "harnessSha": "abb61b2586cb347e0a163d4f8feb2acf74e9cee0", - "hostScope": ["tower2", "m5-mbp", "tower3"] + "reason": "Fleet runs on Tower2, M5, Tower3, and Tower1 found Perplexica nonce failures with this model. On the 2026-09-17 Tower1 public-beta candidate, Perplexica returned an apologetic response without the verification phrase while Pixel, OpenCode, LiteLLM, Open WebUI, and ODS Talk answered on the selected Granite model. A Tower3 comparator answered the same harmless payload after restoring Qwen3.5-27B. Keep Granite 3.2 2B out of all-app release coverage on these hosts until Perplexica passes a real revalidation.", + "evidence": "fleet-test/runs/2026-07-23T21-41-59Z-release-product-cf6dffb8e8bb-harness-47eedd6c0b40-hosts-six-scope-6cycle-switchboard/model-ui/cycle-005/tower2; fleet-test/runs/2026-07-23T21-41-59Z-release-product-cf6dffb8e8bb-harness-47eedd6c0b40-hosts-six-scope-6cycle-switchboard/model-ui/cycle-005/m5-mbp; ods-public-beta-fleet-green-20260911/release-four-host-2f985137-abb61b2-r46/model-ui/cycle-006/tower3/model-ui.json; Tower3 Perplexica live comparator 2026-09-15T16:59Z; ods-public-beta-fleet-green-20260911/release-tower1-pr5579-afe7408-ff56b22-r208/model-ui/cycle-006/tower1/model-ui.json", + "recordedAt": "2026-09-17T01:19:42Z", + "productSha": "afe74085e572c167f44563efc68476f1a910e179", + "harnessSha": "ff56b224a69d8b955a23fd24af8407c9ccf6b4e8", + "hostScope": ["tower2", "m5-mbp", "tower3", "tower1"] }, "hermes_talk": { "status": "unsupported_until_revalidated", diff --git a/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py b/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py index cfc6a16658..96ea40a739 100644 --- a/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py +++ b/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py @@ -588,7 +588,7 @@ def test_real_catalog_gemma_perplexica_block_is_global(): assert tower2["perplexica"]["status"] == "unsupported_until_revalidated" -def test_real_catalog_granite32_perplexica_block_includes_tower3_after_live_failure(): +def test_real_catalog_granite32_perplexica_block_includes_tower1_and_tower3_after_live_failure(): by_id = {model["id"]: model for model in _official_model_catalog()} model = by_id["granite3.2-2b-instruct-q4"] @@ -612,13 +612,19 @@ def test_real_catalog_granite32_perplexica_block_includes_tower3_after_live_fail model, runtime_context={"host": "tower3", "hosts": ["tower3"]}, ) + tower1 = model_app_compatibility( + model, + runtime_context={"host": "tower1", "hosts": ["tower1"]}, + ) assert windows_laptop["perplexica"]["status"] == "unknown" assert strix_halo["perplexica"]["status"] == "unknown" assert tower2["perplexica"]["status"] == "unsupported_until_revalidated" assert m5_mbp["perplexica"]["status"] == "unsupported_until_revalidated" assert tower3["perplexica"]["status"] == "unsupported_until_revalidated" + assert tower1["perplexica"]["status"] == "unsupported_until_revalidated" assert "Tower3" in tower3["perplexica"]["reason"] + assert "Tower1" in tower1["perplexica"]["reason"] def test_real_catalog_smollm3_perplexica_block_is_global(): From 471bc50df3b3a127aa83ac208c7220e19493d8f6 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 16 Sep 2026 22:23:54 -0400 Subject: [PATCH 04/52] fix(pixel): tolerate validated interrupted transition temp on uninstall --- ods/lib/pixel-uninstall.sh | 10 ++++++++++ ods/tests/test-pixel-uninstall.sh | 33 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/ods/lib/pixel-uninstall.sh b/ods/lib/pixel-uninstall.sh index e121a72c4b..fde0ef6697 100644 --- a/ods/lib/pixel-uninstall.sh +++ b/ods/lib/pixel-uninstall.sh @@ -241,10 +241,20 @@ state_limits = { "provider-verified.json": 512 * 1024, "provider-service-environment.json": 1024 * 1024, } +# A hard stop between mkstemp and os.replace can leave an incomplete +# root-owned bridge write behind. It is not a receipt or pending transaction, +# but must be recognized narrowly so it cannot strand an otherwise safe +# uninstall. Python tempfile uses eight [a-z0-9_] characters here. +abandoned_state_temp = re.compile(r"\.transition-[a-z0-9_]{8}\Z") provider_managed = None if present(state_root): directory(state_root, root_uid, root_gid, exact_mode=0o700) for child in state_root.iterdir(): + if abandoned_state_temp.fullmatch(child.name): + info = regular(child, root_uid, root_gid, 8 * 1024 * 1024, private=True) + if stat.S_IMODE(info.st_mode) != 0o600: + raise SystemExit(f"unsafe managed Pixel access temp file: {child}") + continue if child.name not in state_limits: raise SystemExit(f"unexpected Pixel access state: {child.name}") regular(child, root_uid, root_gid, state_limits[child.name], private=True) diff --git a/ods/tests/test-pixel-uninstall.sh b/ods/tests/test-pixel-uninstall.sh index 309c4ce5eb..1a3154316e 100755 --- a/ods/tests/test-pixel-uninstall.sh +++ b/ods/tests/test-pixel-uninstall.sh @@ -1846,6 +1846,39 @@ else fail "verified access coordinator could not be removed" fi +write_access_fixture +printf '%s' '{"interrupted":' > "$ACCESS_STATE/.transition-abc123_4" +chmod 0600 "$ACCESS_STATE/.transition-abc123_4" +if ods_pixel_uninstall_managed "$INSTALL_DIR" "$HOME_DIR" \ + && [[ ! -e "$ACCESS_STATE" && ! -e "$ETC_DIR/pixel-access.json" ]]; then + pass "root-owned interrupted transition temp does not strand managed uninstall" +else + fail "interrupted transition temp stranded managed uninstall" +fi + +write_access_fixture +printf '%s' '{"interrupted":' > "$ACCESS_STATE/.transition-abc123_4" +chmod 0644 "$ACCESS_STATE/.transition-abc123_4" +if ods_pixel_uninstall_managed "$INSTALL_DIR" "$HOME_DIR"; then + fail "non-private interrupted transition temp was accepted" +else + [[ -e "$ACCESS_STATE/.transition-abc123_4" \ + && -e "$ACCESS_STATE/service-baseline.json" && ! -s "$SYSTEMCTL_LOG" ]] \ + && pass "non-private transition temp fails before service mutation" \ + || fail "non-private transition temp caused partial cleanup" +fi + +write_access_fixture +ln -s "$ETC_DIR/pixel-access.json" "$ACCESS_STATE/.transition-abc123_4" +if ods_pixel_uninstall_managed "$INSTALL_DIR" "$HOME_DIR"; then + fail "symlinked interrupted transition temp was accepted" +else + [[ -L "$ACCESS_STATE/.transition-abc123_4" \ + && -e "$ACCESS_STATE/service-baseline.json" && ! -s "$SYSTEMCTL_LOG" ]] \ + && pass "unsafe transition temp fails before service mutation" \ + || fail "unsafe transition temp caused partial cleanup" +fi + for scenario in foreign modified_unit modified_program relay_key state_symlink pending_transition stop_failure still_active; do write_access_fixture case "$scenario" in From 0602e95e0c99c5bebd89b1e5441469a3f5d051c5 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 04:49:23 -0400 Subject: [PATCH 05/52] Scope Granite H-Tiny OpenCode warning to Tower1 --- ods/config/model-library.json | 16 +++++++++++++++- ods/tests/test-model-library-coverage.py | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/ods/config/model-library.json b/ods/config/model-library.json index 447a371fbd..c106e5430e 100644 --- a/ods/config/model-library.json +++ b/ods/config/model-library.json @@ -356,6 +356,19 @@ "tokens_per_sec_estimate": 80, "llm_model_name": "granite-4.0-h-tiny", "llama_server_image": null, + "app_compatibility": { + "opencode": { + "status": "unsupported_until_revalidated", + "label": "OpenCode revalidation required", + "reason": "In the 2026-09-17 Tower1 exact-head release cycle, OpenCode's build agent issued a read tool call against a fabricated path instead of returning its requested verification phrase on Granite 4.0 H-Tiny. The same installed OpenCode path answered on the baseline and neighboring Granite 4.0 models; Pixel, ODS Talk, LiteLLM, Open WebUI, and Perplexica passed on H-Tiny with exact model-route evidence. Keep this model out of Tower1 all-app release coverage until OpenCode behavior is revalidated.", + "evidence": "ods-public-beta-fleet-green-20260911/release-tower1-pr5579-471bc50-135daf6-r216/model-ui/cycle-002/tower1/model-ui.json", + "recordedAt": "2026-09-17T08:46:03Z", + "productSha": "471bc50df3b3a127aa83ac208c7220e19493d8f6", + "harnessSha": "135daf604399fe757e8364d3721767a22bafe541", + "hostScope": ["tower1"], + "expiresAt": "2026-10-17T00:00:00Z" + } + }, "install_recommendation": false }, { @@ -784,7 +797,8 @@ "recordedAt": "2026-09-17T01:19:42Z", "productSha": "afe74085e572c167f44563efc68476f1a910e179", "harnessSha": "ff56b224a69d8b955a23fd24af8407c9ccf6b4e8", - "hostScope": ["tower2", "m5-mbp", "tower3", "tower1"] + "hostScope": ["tower2", "m5-mbp", "tower3", "tower1"], + "expiresAt": "2026-10-17T00:00:00Z" }, "hermes_talk": { "status": "unsupported_until_revalidated", diff --git a/ods/tests/test-model-library-coverage.py b/ods/tests/test-model-library-coverage.py index d7b6eaec17..c5b044163f 100644 --- a/ods/tests/test-model-library-coverage.py +++ b/ods/tests/test-model-library-coverage.py @@ -477,6 +477,20 @@ def test_granite32_2b_is_direct_chat_only_after_windows_talk_timeout(): assert not _agent_viable_for_release(model, host="windows-laptop") +def test_granite4_h_tiny_opencode_warning_is_scoped_to_tower1(): + catalog = json.loads(CATALOG.read_text(encoding="utf-8")) + by_id = {model["id"]: model for model in catalog["models"]} + + model = by_id["granite4.0-h-tiny-q4"] + opencode = model["app_compatibility"]["opencode"] + + assert opencode["status"] == "unsupported_until_revalidated" + assert opencode["hostScope"] == ["tower1"] + assert "cycle-002/tower1/model-ui.json" in opencode["evidence"] + assert not _agent_viable_for_release(model, host="tower1") + assert _agent_viable_for_release(model, host="tower3") + + def test_granite4_h_350m_is_not_agent_viable_after_talk_probe_failure(): catalog = json.loads(CATALOG.read_text(encoding="utf-8")) by_id = {model["id"]: model for model in catalog["models"]} From 0a59db87e7d52db970878d0279e3467d0da9ea4e Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 07:22:55 -0400 Subject: [PATCH 06/52] fix(pixel): keep idle local-model streams alive --- .../services/dashboard-api/routers/pixel.py | 16 ++++++ .../dashboard-api/tests/test_pixel.py | 51 +++++++++++++++++++ .../tests/test_pixel_result_delivery.py | 34 +++++++++++++ 3 files changed, 101 insertions(+) diff --git a/ods/extensions/services/dashboard-api/routers/pixel.py b/ods/extensions/services/dashboard-api/routers/pixel.py index 1abd6f8912..88ddf40d96 100644 --- a/ods/extensions/services/dashboard-api/routers/pixel.py +++ b/ods/extensions/services/dashboard-api/routers/pixel.py @@ -12,6 +12,7 @@ import logging import os import re +import time from pathlib import Path from typing import AsyncIterator, Literal from urllib.parse import urlparse @@ -37,6 +38,8 @@ _MODEL = "pixel/default" _CHAT_STREAM_TIMEOUT_SECONDS = 2040.0 _CLIENT_DISCONNECT_POLL_SECONDS = 0.25 +_STREAM_KEEPALIVE_SECONDS = 15.0 +_STREAM_KEEPALIVE = b": pixel working\n\n" _CLIENT_CANCEL_TIMEOUT_SECONDS = 7.0 _MAX_KEY_LENGTH = 4096 _MAX_STATUS_BYTES = 64 * 1024 @@ -710,6 +713,7 @@ def release(finished): async def subscribe(): after = -1 + last_sent = time.monotonic() while True: # Snapshot terminal state before yielding any bytes. Sending a chunk # can suspend this subscriber while the producer commits its tail. @@ -718,10 +722,17 @@ async def subscribe(): for chunk in store.chunks(identity, after): after = chunk["sequence"] yield chunk["data"] + last_sent = time.monotonic() if row is None or row["state"] != "active": return if await request.is_disconnected(): return + if time.monotonic() - last_sent >= _STREAM_KEEPALIVE_SECONDS: + # A CPU-backed local model can spend minutes in prompt prefill. + # Keep the subscriber alive without inventing an answer or + # persisting transport-only comments in the result receipt. + yield _STREAM_KEEPALIVE + last_sent = time.monotonic() # Subscriber disposal never cancels the independent bounded producer. await asyncio.sleep(_CLIENT_DISCONNECT_POLL_SECONDS) @@ -854,6 +865,7 @@ async def _iter_upstream_chunks( """Yield upstream bytes while promptly observing a silent client exit.""" iterator = upstream.aiter_bytes().__aiter__() pending: asyncio.Task[bytes] | None = None + last_sent = time.monotonic() try: while True: pending = asyncio.create_task(anext(iterator)) @@ -866,12 +878,16 @@ async def _iter_upstream_chunks( break if await request.is_disconnected(): raise _ClientDisconnected + if time.monotonic() - last_sent >= _STREAM_KEEPALIVE_SECONDS: + yield _STREAM_KEEPALIVE + last_sent = time.monotonic() try: chunk = pending.result() except StopAsyncIteration: return pending = None yield chunk + last_sent = time.monotonic() finally: if pending is not None and not pending.done(): pending.cancel() diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel.py b/ods/extensions/services/dashboard-api/tests/test_pixel.py index eae19ea08a..25604ff5ce 100644 --- a/ods/extensions/services/dashboard-api/tests/test_pixel.py +++ b/ods/extensions/services/dashboard-api/tests/test_pixel.py @@ -565,6 +565,57 @@ async def test_chat_forwards_exact_body_and_narrow_edge_key_only(): assert pixel_runtime_state._local_pixel_stream_active() is False +@pytest.mark.asyncio +async def test_chat_keeps_silent_local_inference_stream_alive_without_faking_answer(monkeypatch): + first = b'data: {"choices":[{"delta":{"content":"first"}}]}\n\n' + done = b"data: [DONE]\n\n" + + class SlowResponse(FakeResponse): + async def aiter_bytes(self): + yield first + await asyncio.sleep(0.08) + yield done + + monkeypatch.setattr(pixel, "_STREAM_KEEPALIVE_SECONDS", 0.02) + monkeypatch.setattr(pixel, "_CLIENT_DISCONNECT_POLL_SECONDS", 0.005) + body = pixel.ChatStreamRequest.model_validate( + {"chat_id": "slow_cpu", "messages": [{"role": "user", "content": "hello"}]} + ) + upstream = SlowResponse(content_type="text/event-stream") + with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(upstream)): + response = await pixel.pixel_chat_stream(ConnectedRequest(), body) + streamed = await stream_body(response) + + assert streamed.startswith(first) + assert streamed.endswith(done) + assert pixel._STREAM_KEEPALIVE in streamed[len(first):-len(done)] + assert streamed.count(b"data: [DONE]") == 1 + assert pixel_runtime_state._local_pixel_stream_active() is False + + +@pytest.mark.asyncio +async def test_chat_keepalive_before_first_upstream_byte_is_only_a_comment(monkeypatch): + class SlowFirstResponse(FakeResponse): + async def aiter_bytes(self): + await asyncio.sleep(0.08) + yield b'data: {"choices":[{"delta":{"content":"ready"}}]}\n\n' + yield b'data: [DONE]\n\n' + + monkeypatch.setattr(pixel, "_STREAM_KEEPALIVE_SECONDS", 0.02) + monkeypatch.setattr(pixel, "_CLIENT_DISCONNECT_POLL_SECONDS", 0.005) + body = pixel.ChatStreamRequest.model_validate( + {"chat_id": "slow_first_byte", "messages": [{"role": "user", "content": "hello"}]} + ) + upstream = SlowFirstResponse(content_type="text/event-stream") + with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(upstream)): + response = await pixel.pixel_chat_stream(ConnectedRequest(), body) + streamed = await stream_body(response) + + assert streamed.startswith(pixel._STREAM_KEEPALIVE) + assert b'ready' in streamed + assert streamed.endswith(b'data: [DONE]\n\n') + + @pytest.mark.asyncio async def test_chat_rejects_before_opening_edge_when_stream_capacity_is_full(monkeypatch): body = pixel.ChatStreamRequest.model_validate( diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel_result_delivery.py b/ods/extensions/services/dashboard-api/tests/test_pixel_result_delivery.py index 8615bd40ed..8c64cbc28c 100644 --- a/ods/extensions/services/dashboard-api/tests/test_pixel_result_delivery.py +++ b/ods/extensions/services/dashboard-api/tests/test_pixel_result_delivery.py @@ -53,3 +53,37 @@ async def cancel(*args): assert await stream_body(replay) == retained asyncio.run(run()) + + +def test_retained_subscriber_keepalive_is_not_part_of_durable_reply(store, monkeypatch): + async def run(): + first = b'data: {"choices":[{"delta":{"content":"First "}}]}\n\n' + last = b'data: {"choices":[{"delta":{"content":"last"}}]}\n\n' + + class SlowUpstream(FakeResponse): + async def aiter_bytes(self): + yield first + await asyncio.sleep(0.08) + yield last + yield b'data: [DONE]\n\n' + + monkeypatch.setattr(pixel, '_STREAM_KEEPALIVE_SECONDS', 0.02) + monkeypatch.setattr(pixel, '_CLIENT_DISCONNECT_POLL_SECONDS', 0.005) + monkeypatch.setattr( + pixel.httpx, 'AsyncClient', + lambda **kw: FakeClient(SlowUpstream(content_type='text/event-stream')), + ) + response = await pixel.pixel_chat_stream(ConnectedRequest(), body(), OWNER) + streamed = await stream_body(response) + retained = b''.join(row['data'] for row in store.chunks(IDENTITY)) + + assert first in streamed and last in streamed + assert pixel._STREAM_KEEPALIVE in streamed + assert pixel._STREAM_KEEPALIVE not in retained + assert b'First ' in retained and b'last' in retained + assert retained.count(b'data: [DONE]') == 1 + assert store.get(IDENTITY)['state'] == 'complete' + replay = await pixel.pixel_chat_stream(ConnectedRequest(), body(), OWNER) + assert await stream_body(replay) == retained + + asyncio.run(run()) From 54999a640818c55651742bdfc14696ff37162bf3 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 08:50:58 -0400 Subject: [PATCH 07/52] fix(pixel): avoid keepalive comments inside partial SSE lines --- .../services/dashboard-api/routers/pixel.py | 10 +++++-- .../dashboard-api/tests/test_pixel.py | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/ods/extensions/services/dashboard-api/routers/pixel.py b/ods/extensions/services/dashboard-api/routers/pixel.py index 88ddf40d96..f58917ebd5 100644 --- a/ods/extensions/services/dashboard-api/routers/pixel.py +++ b/ods/extensions/services/dashboard-api/routers/pixel.py @@ -14,7 +14,7 @@ import re import time from pathlib import Path -from typing import AsyncIterator, Literal +from typing import AsyncIterator, Callable, Literal from urllib.parse import urlparse import httpx @@ -861,6 +861,7 @@ def _edge_chat_body(body, messages): async def _iter_upstream_chunks( upstream: httpx.Response, request: Request, + can_emit_keepalive: Callable[[], bool], ) -> AsyncIterator[bytes]: """Yield upstream bytes while promptly observing a silent client exit.""" iterator = upstream.aiter_bytes().__aiter__() @@ -878,7 +879,10 @@ async def _iter_upstream_chunks( break if await request.is_disconnected(): raise _ClientDisconnected - if time.monotonic() - last_sent >= _STREAM_KEEPALIVE_SECONDS: + # A comment is safe only between complete SSE lines. The + # caller may be holding an upstream fragment without a newline; + # injecting a comment there would corrupt that data line. + if can_emit_keepalive() and time.monotonic() - last_sent >= _STREAM_KEEPALIVE_SECONDS: yield _STREAM_KEEPALIVE last_sent = time.monotonic() try: @@ -993,7 +997,7 @@ async def stream() -> AsyncIterator[bytes]: try: async with async_timeout(_CHAT_STREAM_TIMEOUT_SECONDS): buffered = bytearray() - async for chunk in _iter_upstream_chunks(upstream, request): + async for chunk in _iter_upstream_chunks(upstream, request, lambda: not buffered): buffered.extend(chunk) while True: newline = buffered.find(b"\n") diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel.py b/ods/extensions/services/dashboard-api/tests/test_pixel.py index 25604ff5ce..65741cd6b9 100644 --- a/ods/extensions/services/dashboard-api/tests/test_pixel.py +++ b/ods/extensions/services/dashboard-api/tests/test_pixel.py @@ -616,6 +616,35 @@ async def aiter_bytes(self): assert streamed.endswith(b'data: [DONE]\n\n') +@pytest.mark.asyncio +async def test_chat_keepalive_never_splits_an_upstream_sse_line(monkeypatch): + first = b'data: {"choices":[{"delta":{"content":"par' + last = b'tial"}}]}\n\n' + done = b'data: [DONE]\n\n' + + class FragmentedResponse(FakeResponse): + async def aiter_bytes(self): + yield first + await asyncio.sleep(0.08) + yield last + await asyncio.sleep(0.08) + yield done + + monkeypatch.setattr(pixel, "_STREAM_KEEPALIVE_SECONDS", 0.02) + monkeypatch.setattr(pixel, "_CLIENT_DISCONNECT_POLL_SECONDS", 0.005) + body = pixel.ChatStreamRequest.model_validate( + {"chat_id": "fragmented_cpu", "messages": [{"role": "user", "content": "hello"}]} + ) + upstream = FragmentedResponse(content_type="text/event-stream") + with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(upstream)): + response = await pixel.pixel_chat_stream(ConnectedRequest(), body) + streamed = await stream_body(response) + + assert streamed.startswith(first + last) + assert pixel._STREAM_KEEPALIVE in streamed[len(first + last):-len(done)] + assert streamed.endswith(done) + + @pytest.mark.asyncio async def test_chat_rejects_before_opening_edge_when_stream_capacity_is_full(monkeypatch): body = pixel.ChatStreamRequest.model_validate( From efc937edcb1e5c3a1acfa357b0faadfce3a22a62 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 10:38:40 -0400 Subject: [PATCH 08/52] fix(macos): retry transient Colima bridge launchd bootstrap --- ods/installers/macos/lib/bridge-manager.sh | 15 ++++- .../test-macos-bridge-bootstrap-retry.sh | 64 +++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100755 ods/tests/test-macos-bridge-bootstrap-retry.sh diff --git a/ods/installers/macos/lib/bridge-manager.sh b/ods/installers/macos/lib/bridge-manager.sh index 87500d1541..c1032a168a 100755 --- a/ods/installers/macos/lib/bridge-manager.sh +++ b/ods/installers/macos/lib/bridge-manager.sh @@ -59,9 +59,18 @@ macos_configure_port_bridge() { BRIDGE_PLIST_EOF - local bootstrap_err bootstrap_rc - bootstrap_err="$(launchctl bootstrap "gui/$(id -u)" "$plist" 2>&1)" \ - && bootstrap_rc=0 || bootstrap_rc=$? + # bootout is asynchronous: on a forced reinstall launchd can still be + # tearing down the prior service when bootstrap reaches the new plist. + # macOS reports that short-lived collision as rc=5. Retry only that code; + # a malformed plist or another error must still fail the install. + local bootstrap_err bootstrap_rc bootstrap_attempt + for bootstrap_attempt in 1 2 3 4 5 6; do + bootstrap_err="$(launchctl bootstrap "gui/$(id -u)" "$plist" 2>&1)" \ + && bootstrap_rc=0 || bootstrap_rc=$? + [[ "$bootstrap_rc" -eq 0 ]] && break + [[ "$bootstrap_rc" -eq 5 && "$bootstrap_attempt" -lt 6 ]] || break + sleep 1 + done if [[ "$bootstrap_rc" -ne 0 ]]; then ai_err "${description} LaunchAgent failed (rc=${bootstrap_rc}): ${bootstrap_err}" return 1 diff --git a/ods/tests/test-macos-bridge-bootstrap-retry.sh b/ods/tests/test-macos-bridge-bootstrap-retry.sh new file mode 100755 index 0000000000..24935e7597 --- /dev/null +++ b/ods/tests/test-macos-bridge-bootstrap-retry.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# A forced reinstall can race launchd's asynchronous bootout of the old bridge. +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +tmp="$(mktemp -d)" +trap 'rm -rf -- "$tmp"' EXIT +mkdir -p "$tmp/bin" "$tmp/home" "$tmp/install/bin" +: > "$tmp/install/bin/ods-macos-llm-bridge.py" + +cat > "$tmp/bin/launchctl" <<'STUB' +#!/usr/bin/env bash +[[ "$1" == bootstrap ]] || exit 0 +attempts=0 +[[ ! -f "$BRIDGE_TEST_COUNT" ]] || attempts="$(<"$BRIDGE_TEST_COUNT")" +attempts=$((attempts + 1)) +printf '%s\n' "$attempts" > "$BRIDGE_TEST_COUNT" +case "$BRIDGE_TEST_MODE" in + transient) [[ "$attempts" -gt 1 ]] && exit 0; exit 5 ;; + persistent) exit 5 ;; + other) exit 4 ;; +esac +exit 1 +STUB +cat > "$tmp/bin/bridge-python" <<'STUB' +#!/usr/bin/env bash +exit 0 +STUB +chmod +x "$tmp/bin/launchctl" "$tmp/bin/bridge-python" + +export HOME="$tmp/home" PATH="$tmp/bin:$PATH" +export MACOS_BRIDGE_PYTHON="$tmp/bin/bridge-python" +export BRIDGE_TEST_COUNT="$tmp/attempts" BRIDGE_TEST_MODE +ai_err() { printf 'ERROR %s\n' "$*" >&2; } +ai_ok() { :; } +sleep() { :; } +source "$root/installers/macos/lib/bridge-manager.sh" + +run_bridge() { + macos_configure_port_bridge true com.ods.test "$tmp/bridge.plist" "$tmp/bridge.log" \ + "test bridge" 192.168.64.1 8080 8080 192.168.64.2 "$tmp/install" +} + +BRIDGE_TEST_MODE=transient +run_bridge +[[ "$(<"$BRIDGE_TEST_COUNT")" == 2 ]] + +rm -f "$BRIDGE_TEST_COUNT" +BRIDGE_TEST_MODE=persistent +if run_bridge; then + echo 'persistent launchctl rc=5 was accepted' >&2 + exit 1 +fi +[[ "$(<"$BRIDGE_TEST_COUNT")" == 6 ]] + +rm -f "$BRIDGE_TEST_COUNT" +BRIDGE_TEST_MODE=other +if run_bridge; then + echo 'non-retryable launchctl error was accepted' >&2 + exit 1 +fi +[[ "$(<"$BRIDGE_TEST_COUNT")" == 1 ]] + +echo 'macOS bridge bootstrap retry contract passed' From 8448eeecf2ee06ae247920a496a509eda1b9d7c9 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 10:39:01 -0400 Subject: [PATCH 09/52] fix(extensions): expose Aider on CPU fallback installs --- ods/config/extensions-catalog.json | 4 +++- .../library/services/aider/manifest.yaml | 7 ++++--- .../dashboard-api/tests/test_config.py | 18 ++++++++++++++++- .../dashboard-api/tests/test_extensions.py | 20 +++++++++++++++++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/ods/config/extensions-catalog.json b/ods/config/extensions-catalog.json index 1ad42c3658..6d5d266638 100644 --- a/ods/config/extensions-catalog.json +++ b/ods/config/extensions-catalog.json @@ -10,7 +10,9 @@ "gpu_backends": [ "amd", "nvidia", - "apple" + "apple", + "cpu", + "none" ], "compose_file": "compose.yaml", "depends_on": [], diff --git a/ods/extensions/library/services/aider/manifest.yaml b/ods/extensions/library/services/aider/manifest.yaml index 10cf18091b..7123b7b778 100644 --- a/ods/extensions/library/services/aider/manifest.yaml +++ b/ods/extensions/library/services/aider/manifest.yaml @@ -13,9 +13,9 @@ service: health: "" type: docker startup_check: false # one-shot CLI tool; container exits 0 by design - # Aider is a one-shot CLI container and does not require GPU access. `none` - # keeps it installable when ODS selects the CPU fallback backend. - gpu_backends: [amd, nvidia, apple, none] + # Aider is a one-shot CLI container and does not require GPU access. ODS + # writes GPU_BACKEND=cpu for its fallback; retain legacy `none` as well. + gpu_backends: [amd, nvidia, apple, cpu, none] compose_file: compose.yaml category: optional depends_on: [] @@ -38,6 +38,7 @@ features: description: AI pair programming in your terminal icon: Terminal category: development + gpu_backends: [amd, nvidia, apple, cpu, none] requirements: services: [aider] vram_gb: 0 diff --git a/ods/extensions/services/dashboard-api/tests/test_config.py b/ods/extensions/services/dashboard-api/tests/test_config.py index aeb35b49e2..50f8128b34 100644 --- a/ods/extensions/services/dashboard-api/tests/test_config.py +++ b/ods/extensions/services/dashboard-api/tests/test_config.py @@ -1,5 +1,6 @@ """Tests for config.py — manifest loading and service discovery.""" +import json import logging from pathlib import Path @@ -41,7 +42,7 @@ def test_bundled_llama_server_is_discoverable_on_cpu_fallback(): assert all("cpu" in feature["gpu_backends"] for feature in manifest["features"]) -def test_aider_library_extension_is_discoverable_on_cpu_fallback(): +def test_aider_library_extension_is_discoverable_on_cpu_fallback(tmp_path): manifest_path = ( Path(__file__).resolve().parents[3] / "library" @@ -51,7 +52,22 @@ def test_aider_library_extension_is_discoverable_on_cpu_fallback(): ) manifest = config.yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + assert "cpu" in manifest["service"]["gpu_backends"] assert "none" in manifest["service"]["gpu_backends"] + assert all("cpu" in feature["gpu_backends"] for feature in manifest["features"]) + catalog_path = Path(__file__).resolve().parents[4] / "config" / "extensions-catalog.json" + catalog = json.loads(catalog_path.read_text(encoding="utf-8")) + aider = next(ext for ext in catalog["extensions"] if ext["id"] == "aider") + assert {"cpu", "none"}.issubset(aider["gpu_backends"]) + + installed = tmp_path / "aider" + installed.mkdir() + (installed / "manifest.yaml").write_text(manifest_path.read_text(encoding="utf-8")) + (installed / "compose.yaml").write_text("services:\n aider:\n image: test/aider\n") + services, features, errors = load_extension_manifests(tmp_path, "cpu") + assert errors == [] + assert "aider" in services + assert any(feature["id"] == "ai-pair-programming" for feature in features) def test_manifest_loader_rejects_pathological_nesting(tmp_path): diff --git a/ods/extensions/services/dashboard-api/tests/test_extensions.py b/ods/extensions/services/dashboard-api/tests/test_extensions.py index ce72c6ae30..6021d87b31 100644 --- a/ods/extensions/services/dashboard-api/tests/test_extensions.py +++ b/ods/extensions/services/dashboard-api/tests/test_extensions.py @@ -188,6 +188,26 @@ def test_catalog_gpu_compatible_filter(self, test_client, monkeypatch, tmp_path) assert "compat" in ids assert "incompat" not in ids + def test_aider_is_visible_and_installable_on_cpu_fallback(self, test_client, monkeypatch, tmp_path): + """The CPU fallback must not hide Aider's zero-VRAM CLI card.""" + catalog_path = Path(__file__).resolve().parents[4] / "config" / "extensions-catalog.json" + catalog = json.loads(catalog_path.read_text(encoding="utf-8")) + aider = next(ext for ext in catalog["extensions"] if ext["id"] == "aider") + gpu_only = _make_catalog_ext("gpu-only", "GPU only", gpu_backends=["nvidia"]) + _patch_extensions_config(monkeypatch, [aider, gpu_only], gpu_backend="cpu", tmp_path=tmp_path) + library_dir = tmp_path / "lib" / "aider" + library_dir.mkdir(parents=True) + (library_dir / "compose.yaml").write_text("services:\n aider:\n image: test/aider\n") + + with patch("helpers.get_all_services", new_callable=AsyncMock, return_value=[]): + resp = test_client.get("/api/extensions/catalog", headers=test_client.auth_headers) + + assert resp.status_code == 200 + by_id = {ext["id"]: ext for ext in resp.json()["extensions"]} + assert by_id["aider"]["status"] == "not_installed" + assert by_id["aider"]["installable"] is True + assert by_id["gpu-only"]["status"] == "incompatible" + def test_catalog_summary_counts(self, test_client, monkeypatch, tmp_path): """Summary counts correctly reflect extension statuses.""" catalog = [ From 3db1b08b2ea60b8b3f5ecbe884ae8c09db21040c Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 11:46:01 -0400 Subject: [PATCH 10/52] fix(macos): preserve Langfuse data credentials on forced reinstall --- ods/installers/macos/lib/env-generator.sh | 37 ++++++++++++++++++----- ods/tests/test-macos-env-schema.sh | 34 +++++++++++++++++++++ 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/ods/installers/macos/lib/env-generator.sh b/ods/installers/macos/lib/env-generator.sh index 8dba3182c3..2f021ea41d 100755 --- a/ods/installers/macos/lib/env-generator.sh +++ b/ods/installers/macos/lib/env-generator.sh @@ -421,9 +421,9 @@ generate_ods_env() { opencode_password=$(new_secure_base64 16) local searxng_secret searxng_secret=$(new_secure_hex 32) - # Langfuse (LLM Observability) - # NOTE: macOS env-generator always regenerates secrets (no merge logic). - # If reinstalling with existing Langfuse data, run: rm -rf data/langfuse/ + # Langfuse (LLM Observability). A forced reinstall may regenerate other + # secrets, but data-bound Langfuse credentials must stay paired with its + # persisted PostgreSQL, ClickHouse, Redis, and MinIO state. local langfuse_nextauth_secret langfuse_nextauth_secret=$(new_secure_hex 32) local langfuse_salt @@ -448,6 +448,32 @@ generate_ods_env() { langfuse_init_project_id=$(new_secure_hex 16) local langfuse_init_user_password langfuse_init_user_password=$(new_secure_hex 16) + if [[ -f "${install_dir}/data/langfuse/postgres/PG_VERSION" ]]; then + local langfuse_key + for langfuse_key in \ + LANGFUSE_NEXTAUTH_SECRET LANGFUSE_SALT LANGFUSE_ENCRYPTION_KEY \ + LANGFUSE_DB_PASSWORD LANGFUSE_CLICKHOUSE_PASSWORD LANGFUSE_REDIS_PASSWORD \ + LANGFUSE_MINIO_ACCESS_KEY LANGFUSE_MINIO_SECRET_KEY \ + LANGFUSE_PROJECT_PUBLIC_KEY LANGFUSE_PROJECT_SECRET_KEY \ + LANGFUSE_INIT_PROJECT_ID LANGFUSE_INIT_USER_PASSWORD; do + if [[ -z "$(read_env_value "$env_path" "$langfuse_key")" ]]; then + printf 'Existing Langfuse database requires %s in the previous .env; refusing to rotate persisted credentials.\n' "$langfuse_key" >&2 + return 1 + fi + done + langfuse_nextauth_secret=$(read_env_value "$env_path" LANGFUSE_NEXTAUTH_SECRET) + langfuse_salt=$(read_env_value "$env_path" LANGFUSE_SALT) + langfuse_encryption_key=$(read_env_value "$env_path" LANGFUSE_ENCRYPTION_KEY) + langfuse_db_password=$(read_env_value "$env_path" LANGFUSE_DB_PASSWORD) + langfuse_clickhouse_password=$(read_env_value "$env_path" LANGFUSE_CLICKHOUSE_PASSWORD) + langfuse_redis_password=$(read_env_value "$env_path" LANGFUSE_REDIS_PASSWORD) + langfuse_minio_access_key=$(read_env_value "$env_path" LANGFUSE_MINIO_ACCESS_KEY) + langfuse_minio_secret_key=$(read_env_value "$env_path" LANGFUSE_MINIO_SECRET_KEY) + langfuse_project_public_key=$(read_env_value "$env_path" LANGFUSE_PROJECT_PUBLIC_KEY) + langfuse_project_secret_key=$(read_env_value "$env_path" LANGFUSE_PROJECT_SECRET_KEY) + langfuse_init_project_id=$(read_env_value "$env_path" LANGFUSE_INIT_PROJECT_ID) + langfuse_init_user_password=$(read_env_value "$env_path" LANGFUSE_INIT_USER_PASSWORD) + fi # Colima's user-mode host.docker.internal route can become unreachable # under load. The orchestrator enables its private vmnet address first; # bridge loopback-only host services through that scoped interface. @@ -695,10 +721,7 @@ N8N_WEBHOOK_URL=http://localhost:5678 TIMEZONE=${tz} #=== Langfuse (LLM Observability) === -# NOTE: this value is only written on first install or --force (the macOS -# env-generator early-returns when .env already exists). Users who re-run -# ./install-macos.sh --langfuse on an existing install should instead use -# post-install: 'ods enable langfuse'. +# Existing Langfuse state keeps its data-bound secrets even with --force. LANGFUSE_ENABLED=${ENABLE_LANGFUSE:-false} LANGFUSE_NEXTAUTH_SECRET=${langfuse_nextauth_secret} LANGFUSE_SALT=${langfuse_salt} diff --git a/ods/tests/test-macos-env-schema.sh b/ods/tests/test-macos-env-schema.sh index f7ada7a1bd..3808f67969 100755 --- a/ods/tests/test-macos-env-schema.sh +++ b/ods/tests/test-macos-env-schema.sh @@ -95,3 +95,37 @@ for tier in 1 CLOUD; do fi pass "tier $tier: generated .env validates against .env.schema.json" done + +# A forced reinstall must not rotate credentials already bound to a persisted +# Langfuse database. Other install secrets may still rotate under --force. +langfuse_dir="$TMP_DIR/langfuse-force" +generate_env 1 "$langfuse_dir" +mkdir -p "$langfuse_dir/data/langfuse/postgres" +printf '16\n' > "$langfuse_dir/data/langfuse/postgres/PG_VERSION" +old_langfuse="$(grep '^LANGFUSE_' "$langfuse_dir/.env")" +old_dashboard_key="$(grep '^DASHBOARD_API_KEY=' "$langfuse_dir/.env")" +generate_env 1 "$langfuse_dir" +[[ "$(grep '^LANGFUSE_' "$langfuse_dir/.env")" == "$old_langfuse" ]] \ + || fail 'forced reinstall rotated persisted Langfuse credentials' +[[ "$(grep '^DASHBOARD_API_KEY=' "$langfuse_dir/.env")" != "$old_dashboard_key" ]] \ + || fail 'forced reinstall did not rotate an unbound install secret' +pass 'forced reinstall preserves persisted Langfuse credentials' + +missing_env_dir="$TMP_DIR/langfuse-without-env" +mkdir -p "$missing_env_dir/data/langfuse/postgres" +printf '16\n' > "$missing_env_dir/data/langfuse/postgres/PG_VERSION" +if (generate_env 1 "$missing_env_dir") >/dev/null 2>&1; then + fail 'persisted Langfuse database accepted a missing prior .env' +fi +[[ ! -f "$missing_env_dir/.env" ]] \ + || fail 'missing prior Langfuse credentials still produced a new .env' +pass 'persisted Langfuse database fails closed without prior credentials' + +sed '/^LANGFUSE_DB_PASSWORD=/d' "$langfuse_dir/.env" > "$langfuse_dir/.env.missing-key" +mv "$langfuse_dir/.env.missing-key" "$langfuse_dir/.env" +if (generate_env 1 "$langfuse_dir") >/dev/null 2>&1; then + fail 'persisted Langfuse database accepted a missing password' +fi +[[ "$(grep -c '^LANGFUSE_DB_PASSWORD=' "$langfuse_dir/.env" || true)" -eq 0 ]] \ + || fail 'rejected Langfuse credentials were overwritten' +pass 'persisted Langfuse database fails closed on an incomplete prior .env' From 937d9977f70f10c0154f9f2f2bc7ce01ebe519f1 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 12:37:36 -0400 Subject: [PATCH 11/52] fix(pixel): bind external Lemonade model on CPU WSL hosts --- ods/installers/lib/pixel-host-install.sh | 6 ++++++ ods/tests/test-pixel-host-install.sh | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/ods/installers/lib/pixel-host-install.sh b/ods/installers/lib/pixel-host-install.sh index 077806fc70..8df139082d 100644 --- a/ods/installers/lib/pixel-host-install.sh +++ b/ods/installers/lib/pixel-host-install.sh @@ -2998,6 +2998,12 @@ _ods_pixel_runtime_model_identity() { if [[ -n "${EXTERNAL_LLM_URL:-}" ]]; then model="${EXTERNAL_LLM_MODEL:-}" [[ -n "$model" ]] || return 1 + elif [[ "${LEMONADE_EXTERNAL:-false}" == true ]]; then + # WSL can attach to a Windows-hosted Lemonade server while its Linux + # hardware detector correctly reports CPU. Bind Pixel to the served + # model, not the stale GGUF selected before the external route. + model="${LEMONADE_MODEL:-}" + [[ -n "$model" ]] || return 1 elif [[ "${GPU_BACKEND:-}" == amd \ && "${LLM_BACKEND:-}" == lemonade \ && "${AMD_INFERENCE_RUNTIME:-}" == lemonade ]]; then diff --git a/ods/tests/test-pixel-host-install.sh b/ods/tests/test-pixel-host-install.sh index 0a9a5f784c..c53242c83c 100644 --- a/ods/tests/test-pixel-host-install.sh +++ b/ods/tests/test-pixel-host-install.sh @@ -2067,6 +2067,15 @@ check test "$(GGUF_FILE='My Custom Model (Q4_K_M).gguf' \ AMD_INFERENCE_RUNTIME=lemonade \ LEMONADE_MODEL='extra.My Custom Model (Q4_K_M).gguf' \ _ods_pixel_runtime_model_identity)" = 'extra.My Custom Model (Q4_K_M).gguf' +check test "$(GGUF_FILE='stale-local.gguf' GPU_BACKEND=cpu \ + LEMONADE_EXTERNAL=true LEMONADE_MODEL='Qwen3.6-35B-A3B-GGUF' \ + _ods_pixel_runtime_model_identity)" = 'Qwen3.6-35B-A3B-GGUF' +if GGUF_FILE='stale-local.gguf' GPU_BACKEND=cpu LEMONADE_EXTERNAL=true \ + LEMONADE_MODEL='' _ods_pixel_runtime_model_identity >/dev/null 2>&1; then + fail "external Lemonade runtime identity requires the served model" +else + pass "external Lemonade runtime identity requires the served model" +fi check test "$(EXTERNAL_LLM_URL='http://10.0.2.2:18080' \ EXTERNAL_LLM_MODEL='org/qwen+tools:remote' GGUF_FILE='stale-local.gguf' \ _ods_pixel_runtime_model_identity)" = 'org/qwen+tools:remote' From 770d81697c41127431e3b1fa77aeac52e10264cf Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 15:40:27 -0400 Subject: [PATCH 12/52] fix(wsl): tolerate unavailable Windows interop during RAM detection --- ods/installers/lib/wsl-memory.sh | 28 ++++++++++++++ ods/installers/phases/02-detection.sh | 23 ++--------- ods/tests/test-wsl-host-ram-fallback.sh | 51 +++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 19 deletions(-) create mode 100755 ods/tests/test-wsl-host-ram-fallback.sh diff --git a/ods/installers/lib/wsl-memory.sh b/ods/installers/lib/wsl-memory.sh index 6d0f477170..e845229993 100644 --- a/ods/installers/lib/wsl-memory.sh +++ b/ods/installers/lib/wsl-memory.sh @@ -3,6 +3,34 @@ ODS_WSL_CONTROL_PLANE_HEADROOM_GB_DEFAULT=2 +# Windows interop is optional in WSL. Its executable can be on PATH while the +# WSLInterop binfmt handler is unavailable (Exec format error, exit 126). +# Never make an optional host-RAM lookup abort Linux hardware detection. +ods_wsl_host_ram_kb() { + local host_bytes="" host_kb="" + + if command -v powershell.exe >/dev/null 2>&1; then + host_bytes="$(powershell.exe -NoProfile -Command \ + '(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory' 2>/dev/null \ + | tr -d '\r')" || host_bytes="" + if [[ "$host_bytes" =~ ^[0-9]+$ ]]; then + printf '%s\n' "$((host_bytes / 1024))" + return 0 + fi + fi + + if command -v wmic.exe >/dev/null 2>&1; then + host_kb="$(wmic.exe OS get TotalVisibleMemorySize /value 2>/dev/null \ + | grep -oE '[0-9]+' | sed -n '1p')" || host_kb="" + if [[ "$host_kb" =~ ^[0-9]+$ ]]; then + printf '%s\n' "$host_kb" + return 0 + fi + fi + + return 1 +} + ods_wsl_model_ram_budget() { local vm_ram_gb="${1:-0}" local headroom_gb="${2:-${ODS_WSL_CONTROL_PLANE_HEADROOM_GB:-$ODS_WSL_CONTROL_PLANE_HEADROOM_GB_DEFAULT}}" diff --git a/ods/installers/phases/02-detection.sh b/ods/installers/phases/02-detection.sh index cf74dea83c..d3bd89a4ed 100755 --- a/ods/installers/phases/02-detection.sh +++ b/ods/installers/phases/02-detection.sh @@ -50,13 +50,9 @@ if [[ "${ODS_MODE:-local}" == "cloud" ]]; then GPU_MEMORY_TYPE="none" TIER="CLOUD" if grep -qi microsoft /proc/version 2>/dev/null; then - _wsl_ram_bytes="" - if command -v powershell.exe &>/dev/null; then - _wsl_ram_bytes=$(powershell.exe -NoProfile -Command \ - "(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory" 2>/dev/null | tr -d '\r') - fi - if [[ -n "$_wsl_ram_bytes" && "$_wsl_ram_bytes" =~ ^[0-9]+$ ]]; then - RAM_KB=$((_wsl_ram_bytes / 1024)) + _wsl_host_kb="$(ods_wsl_host_ram_kb)" || _wsl_host_kb="" + if [[ -n "$_wsl_host_kb" ]]; then + RAM_KB="$_wsl_host_kb" else RAM_KB=$(grep MemTotal /proc/meminfo | awk '{print $2}') fi @@ -90,18 +86,7 @@ load_capability_profile || true # reserved value only for coarse tier selection; system_ram_min_gb profiles and # the persisted SYSTEM_RAM_GB contract describe actual addressable VM memory. if grep -qi microsoft /proc/version 2>/dev/null; then - _wsl_ram_kb="" - if command -v powershell.exe &>/dev/null; then - _wsl_ram_bytes=$(powershell.exe -NoProfile -Command \ - "(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory" 2>/dev/null | tr -d '\r') - if [[ -n "$_wsl_ram_bytes" && "$_wsl_ram_bytes" =~ ^[0-9]+$ ]]; then - _wsl_ram_kb=$((_wsl_ram_bytes / 1024)) - fi - fi - if [[ -z "$_wsl_ram_kb" ]] && command -v wmic.exe &>/dev/null; then - _wsl_ram_kb=$(wmic.exe OS get TotalVisibleMemorySize /value 2>/dev/null \ - | grep -oE '[0-9]+' | sed -n '1p') - fi + _wsl_ram_kb="$(ods_wsl_host_ram_kb)" || _wsl_ram_kb="" _wsl_vm_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}') RAM_KB="$_wsl_vm_kb" RAM_GB=$((RAM_KB / 1024 / 1024)) diff --git a/ods/tests/test-wsl-host-ram-fallback.sh b/ods/tests/test-wsl-host-ram-fallback.sh new file mode 100755 index 0000000000..0748c566ce --- /dev/null +++ b/ods/tests/test-wsl-host-ram-fallback.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +tmp="$(mktemp -d)" +trap 'rm -rf -- "$tmp"' EXIT + +cat >"$tmp/powershell.exe" <<'PS' +#!/usr/bin/env bash +if [[ "${ODS_TEST_PS_MODE:-ok}" == fail ]]; then + printf 'Exec format error\n' >&2 + exit 126 +fi +printf '17179869184\r\n' +PS +cat >"$tmp/wmic.exe" <<'WMIC' +#!/usr/bin/env bash +if [[ "${ODS_TEST_WMIC_MODE:-ok}" == fail ]]; then + printf 'Exec format error\n' >&2 + exit 126 +fi +printf 'TotalVisibleMemorySize=33554432\r\n' +WMIC +chmod +x "$tmp/powershell.exe" "$tmp/wmic.exe" + +export PATH="$tmp:$PATH" +source "$root/installers/lib/wsl-memory.sh" + +[[ "$(ods_wsl_host_ram_kb)" == 16777216 ]] +ODS_TEST_PS_MODE=fail +export ODS_TEST_PS_MODE +[[ "$(ods_wsl_host_ram_kb)" == 33554432 ]] +ODS_TEST_WMIC_MODE=fail +export ODS_TEST_WMIC_MODE +if ods_wsl_host_ram_kb >"$tmp/failed.out"; then + printf 'FAIL: unavailable Windows interop supplied host RAM\n' >&2 + exit 1 +fi +[[ ! -s "$tmp/failed.out" ]] + +# This is the installer call shape under set -euo pipefail: a failed optional +# Windows query must leave the caller alive to use Linux /proc/meminfo. +vm_kb=49209324 +host_kb="$(ods_wsl_host_ram_kb)" || host_kb="" +ram_kb="${host_kb:-$vm_kb}" +[[ "$ram_kb" == "$vm_kb" ]] + +phase="$root/installers/phases/02-detection.sh" +[[ "$(grep -Fc 'ods_wsl_host_ram_kb)" || _wsl_' "$phase")" == 2 ]] + +printf 'PASS: WSL host-RAM lookup falls back when Windows interop exits 126\n' From 14bb7963fc52022cde240973dc5841867751bb94 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 15:53:12 -0400 Subject: [PATCH 13/52] fix(pixel): start shared search before plan preflight --- ods/installers/lib/pixel-host-install.sh | 14 ++++++++------ ods/tests/test-pixel-host-install.sh | 4 +++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ods/installers/lib/pixel-host-install.sh b/ods/installers/lib/pixel-host-install.sh index 8df139082d..156a503a43 100644 --- a/ods/installers/lib/pixel-host-install.sh +++ b/ods/installers/lib/pixel-host-install.sh @@ -4597,7 +4597,10 @@ ods_pixel_install_default_agent() { # transition gate. Start the edge before the host ingress is installed; # its transition endpoint is independent of upstream chat readiness, and # the final access reproof below still runs only after ingress is healthy. - local -a pixel_prerequisites=(litellm dashboard-api pixel-edge pixel-model-relay) + # Pixel's plan preflight probes SearXNG even when its agentic web-search + # provider is Parallel. ODS also shares SearXNG with OWUI/Perplexica, so + # a clean install must start it before Pixel plans its host deployment. + local -a pixel_prerequisites=(litellm dashboard-api pixel-edge pixel-model-relay searxng) owner="${PIXEL_SERVICE_USER:-$(ods_pixel_install_owner)}" || return 1 home="$(ods_pixel_owner_home "$owner")" || return 1 pixel_gateway_port="$(_ods_pixel_gateway_port)" || { @@ -4677,8 +4680,7 @@ ods_pixel_install_default_agent() { "$plugin_root/host/native_search.py" --answers-file "$answers" \ --provider "${PIXEL_WEB_SEARCH_PROVIDER:-}")" || return 1 case "$web_search_provider" in - searxng) pixel_prerequisites+=(searxng) ;; - parallel-free) ;; + searxng|parallel-free) ;; *) ai_bad "Pixel returned an invalid native search provider."; return 1 ;; esac ai "Starting the ODS model gateway, control API, and search prerequisites for Pixel review..." @@ -4697,9 +4699,9 @@ ods_pixel_install_default_agent() { fi _ods_pixel_wait_model_gateway "ODS Pixel model relay" "${PIXEL_MODEL_RELAY_PORT:-4006}" \ "${PIXEL_MODEL_RELAY_KEY:-}" "$gateway_alias" 180 - if [[ "$web_search_provider" == searxng ]]; then - _ods_pixel_wait_http "ODS local search" "http://127.0.0.1:${SEARXNG_PORT:-8888}/search?q=pixel-preflight&format=json" 90 '.results | type == "array"' - fi + _ods_pixel_wait_http "ODS local search" \ + "http://127.0.0.1:${SEARXNG_PORT:-8888}/search?q=pixel-preflight&format=json" \ + 90 '.results | type == "array"' _ods_pixel_wait_http "ODS control API" \ "http://127.0.0.1:${DASHBOARD_API_PORT:-3002}/health" 90 diff --git a/ods/tests/test-pixel-host-install.sh b/ods/tests/test-pixel-host-install.sh index c53242c83c..92c148393f 100644 --- a/ods/tests/test-pixel-host-install.sh +++ b/ods/tests/test-pixel-host-install.sh @@ -2287,7 +2287,9 @@ import pathlib,sys text=pathlib.Path(sys.argv[1]).read_text() access_bridge=pathlib.Path(sys.argv[2]).read_text() installer=text[text.index("ods_pixel_install_default_agent() {"):] -assert "local -a pixel_prerequisites=(litellm dashboard-api pixel-edge pixel-model-relay)" in installer +assert "local -a pixel_prerequisites=(litellm dashboard-api pixel-edge pixel-model-relay searxng)" in installer +assert "pixel_prerequisites+=(searxng)" not in installer +assert installer.index("\"ODS local search\"") < installer.index("\"$pixel_root/pixel\" plan") assert "ods_pixel_run_as_owner \"$owner\" \"$home\" curl" in text assert "_ods_pixel_wait_ingress \"$owner\" \"$home\"" in installer assert installer.index("_ods_pixel_wait_ingress \"$owner\" \"$home\"") < installer.index("_ods_pixel_mark_ready \"$owner\" \"$home\"") From 832b7d036a979d3dd970288615afa201ec2f28c6 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 17:16:19 -0400 Subject: [PATCH 14/52] fix(models): keep Lemonade discovery on its physical backend --- .../services/dashboard-api/routers/models.py | 8 ++++++++ .../dashboard-api/tests/test_models.py | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/ods/extensions/services/dashboard-api/routers/models.py b/ods/extensions/services/dashboard-api/routers/models.py index 475e2533d5..2c83a168da 100644 --- a/ods/extensions/services/dashboard-api/routers/models.py +++ b/ods/extensions/services/dashboard-api/routers/models.py @@ -353,6 +353,14 @@ def _strip_llm_api_suffix(base_url: str) -> str: def _configured_llm_base_url(host: str, port: int) -> str: + # LiteLLM's LLM_API_URL is an alias gateway, not the physical Lemonade + # runtime. Model identity and readiness probes must follow the same + # backend endpoint as the installed host-inference route. + if LLM_BACKEND == "lemonade": + for key in ("LEMONADE_CONTAINER_BASE_URL", "LEMONADE_BASE_URL"): + value = read_env_value(key, INSTALL_DIR) + if value: + return _strip_llm_api_suffix(value) for key in ("LLM_URL", "LLM_API_URL", "OLLAMA_URL"): value = read_env_value(key, INSTALL_DIR) if value: diff --git a/ods/extensions/services/dashboard-api/tests/test_models.py b/ods/extensions/services/dashboard-api/tests/test_models.py index 6f73df57f2..ad95ef234d 100644 --- a/ods/extensions/services/dashboard-api/tests/test_models.py +++ b/ods/extensions/services/dashboard-api/tests/test_models.py @@ -1957,6 +1957,26 @@ def test_api_models_falls_back_to_loaded_model_probe(test_client, monkeypatch, t assert payload["models"][0]["performance"]["source"] == "measured_local" +def test_lemonade_model_probe_uses_physical_backend_not_litellm_alias(monkeypatch, tmp_path): + import routers.models as models_router + + values = { + "LLM_API_URL": "http://litellm:4000", + "LEMONADE_CONTAINER_BASE_URL": "http://192.168.0.166:8080", + "LEMONADE_BASE_URL": "http://192.168.0.167:8080", + } + monkeypatch.setattr(models_router, "INSTALL_DIR", str(tmp_path)) + monkeypatch.setattr(models_router, "read_env_value", lambda key, _root: values.get(key)) + monkeypatch.setattr(models_router, "LLM_BACKEND", "lemonade") + + assert models_router._configured_llm_base_url("llama-server", 8080) == "http://192.168.0.166:8080" + values.pop("LEMONADE_CONTAINER_BASE_URL") + assert models_router._configured_llm_base_url("llama-server", 8080) == "http://192.168.0.167:8080" + + monkeypatch.setattr(models_router, "LLM_BACKEND", "llama-server") + assert models_router._configured_llm_base_url("llama-server", 8080) == "http://litellm:4000" + + def test_api_models_marks_installer_configured_model(test_client, monkeypatch, tmp_path): models_router, install_dir, _data_dir = _patch_model_router_paths(monkeypatch, tmp_path) _write_model_library(install_dir, [{ From c85b12d2d604d02b8df464db8a76a6a1343a70d3 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 17:42:02 -0400 Subject: [PATCH 15/52] fix(doctor): probe external Lemonade instead of local llama-server --- ods/scripts/ods-doctor.sh | 35 +++++++- .../test-ods-doctor-external-lemonade.sh | 85 +++++++++++++++++++ 2 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 ods/tests/test-ods-doctor-external-lemonade.sh diff --git a/ods/scripts/ods-doctor.sh b/ods/scripts/ods-doctor.sh index 51ea3bc089..023fb44395 100755 --- a/ods/scripts/ods-doctor.sh +++ b/ods/scripts/ods-doctor.sh @@ -200,18 +200,36 @@ LLM_RECOVERY="" _doctor_check_external_llm() { local url="$1" provider="$2" model="$3" local health_path + local lemonade_key="" probe_ok=false LLM_URL="$url" LLM_PROVIDER="${provider:-external}" LLM_MODEL="$model" + LLM_RECOVERY="" + LLM_LOCAL_WARNING="false" case "$provider" in ollama) health_path="/api/tags" ;; lmstudio) health_path="/v1/models" ;; + lemonade) + local api_path="${LEMONADE_API_BASE_PATH:-/api/v1}" + api_path="/${api_path#/}" + health_path="${api_path%/}/models" + lemonade_key="${LEMONADE_API_KEY:-${LEMONADE_ADMIN_API_KEY:-${LITELLM_LEMONADE_API_KEY:-}}}" + ;; *) health_path="/v1/models" ;; # OpenAI-compat fallback esac - if command -v curl >/dev/null 2>&1 && curl -sf --max-time 5 "${url}${health_path}" > /dev/null 2>&1; then + if command -v curl >/dev/null 2>&1; then + if curl -sf --max-time 5 "${url%/}${health_path}" > /dev/null 2>&1; then + probe_ok=true + elif [[ "$provider" == lemonade && -n "$lemonade_key" ]] \ + && curl -sf --max-time 5 -H "Authorization: Bearer ${lemonade_key}" \ + "${url%/}${health_path}" > /dev/null 2>&1; then + probe_ok=true + fi + fi + if [[ "$probe_ok" == true ]]; then LLM_STATUS="ok" log_ok "LLM backend: ${provider:-external} (external) — responding" log_ok " Endpoint : $url" @@ -279,6 +297,19 @@ _doctor_check_llm_backend() { if [ -n "$ext_url" ]; then # External LLM mode — skip llama-server check _doctor_check_external_llm "$ext_url" "$ext_provider" "$ext_model" + elif [[ "${LEMONADE_EXTERNAL:-false}" == "true" && ( "$mode" == "lemonade" || "${LLM_BACKEND:-}" == "lemonade" ) ]]; then + local lemonade_url="${LEMONADE_BASE_URL:-}" + if [[ -n "$lemonade_url" ]]; then + _doctor_check_external_llm "$lemonade_url" lemonade "${LEMONADE_MODEL:-}" + else + LLM_URL="" + LLM_PROVIDER="lemonade" + LLM_MODEL="${LEMONADE_MODEL:-}" + LLM_STATUS="fail" + LLM_RECOVERY="set LEMONADE_BASE_URL to the host-reachable Lemonade endpoint" + log_fail "LLM backend: lemonade (external) — host endpoint missing" + log_info " Recovery : ${LLM_RECOVERY}" + fi elif [[ "$mode" == "cloud" ]]; then local cloud_url="${LLM_API_URL:-}" if [ -n "$cloud_url" ]; then @@ -326,7 +357,7 @@ _doctor_check_llm_backend() { LLM_RECOVERY="" fi else - # Local, hybrid, lemonade modes (or default local) — existing llama-server container check unchanged + # Managed local/hybrid runtimes still use the local container check. _doctor_check_llama_server fi } diff --git a/ods/tests/test-ods-doctor-external-lemonade.sh b/ods/tests/test-ods-doctor-external-lemonade.sh new file mode 100644 index 0000000000..2216e50527 --- /dev/null +++ b/ods/tests/test-ods-doctor-external-lemonade.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source_file="$root/scripts/ods-doctor.sh" +extract_function() { + awk -v signature="^$1[(][)]" ' + $0 ~ signature { in_block = 1 } + in_block { print } + in_block && /^}/ { exit } + ' "$source_file" +} +eval "$(extract_function _doctor_check_external_llm)" +eval "$(extract_function _doctor_check_llm_backend)" + +fail() { printf '[FAIL] %s\n' "$*" >&2; exit 1; } +log_ok() { :; } +log_fail() { :; } +log_info() { :; } +log_warn() { :; } + +curl_calls=0 +curl_rc=0 +curl_args=() +curl() { + curl_calls=$((curl_calls + 1)) + curl_args=("$@") + if [[ " ${curl_args[*]} " == *' Authorization: Bearer sk-fixture-not-real '* ]]; then + return 0 + fi + return "$curl_rc" +} +local_checks=0 +_doctor_check_llama_server() { + local_checks=$((local_checks + 1)) + LLM_STATUS=local +} + +DOCKER_DAEMON=false +EXTERNAL_LLM_URL= +ODS_MODE=lemonade +LLM_BACKEND=lemonade +LEMONADE_EXTERNAL=true +LEMONADE_BASE_URL=http://127.0.0.1:8080 +LEMONADE_API_BASE_PATH=/api/v1 +LEMONADE_MODEL=Qwen3.6-35B-A3B-GGUF +LEMONADE_API_KEY=sk-fixture-not-real +curl_rc=22 +_doctor_check_llm_backend +[[ "$LLM_STATUS" == ok && "$LLM_PROVIDER" == lemonade && "$LLM_MODEL" == "$LEMONADE_MODEL" ]] \ + || fail 'external Lemonade must be reported as the active, healthy LLM backend' +[[ "$curl_calls" == 2 && "$local_checks" == 0 ]] \ + || fail 'doctor must retry a protected external Lemonade route with its configured key' +[[ " ${curl_args[*]} " == *' http://127.0.0.1:8080/api/v1/models '* ]] \ + || fail 'doctor must probe the versioned Lemonade models endpoint' +[[ " ${curl_args[*]} " == *' Authorization: Bearer sk-fixture-not-real '* ]] \ + || fail 'doctor must authenticate a protected Lemonade endpoint' + +unset LEMONADE_API_KEY +unset LEMONADE_ADMIN_API_KEY LITELLM_LEMONADE_API_KEY +curl_rc=0 +before_calls="$curl_calls" +_doctor_check_llm_backend +[[ "$LLM_STATUS" == ok && "$curl_calls" == "$((before_calls + 1))" ]] \ + || fail 'unprotected Lemonade must succeed without a fabricated credential' + +curl_rc=22 +_doctor_check_llm_backend +[[ "$LLM_STATUS" == fail && "$LLM_PROVIDER" == lemonade && "$local_checks" == 0 ]] \ + || fail 'unreachable Lemonade must fail, not fall back to absent llama-server' + +LEMONADE_BASE_URL= +before_calls="$curl_calls" +_doctor_check_llm_backend +[[ "$LLM_STATUS" == fail && "$LLM_PROVIDER" == lemonade && "$curl_calls" == "$before_calls" ]] \ + || fail 'missing external host endpoint must fail closed without a localhost probe' + +LEMONADE_EXTERNAL=false +ODS_MODE=local +LLM_BACKEND=llama-server +_doctor_check_llm_backend +[[ "$LLM_STATUS" == local && "$local_checks" == 1 ]] \ + || fail 'managed local installs must retain the llama-server diagnostic' + +printf '[OK] doctor checks the configured external Lemonade route, not absent local llama-server\n' From 2a471e8abb95503e6e0631888cc0111ac6a869f8 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 17:45:01 -0400 Subject: [PATCH 16/52] test(pixel): ignore transient Git maintenance locks in source assertion --- .../pixel-agent/tests/test_workspace_preview_git.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ods/extensions/services/pixel-agent/tests/test_workspace_preview_git.py b/ods/extensions/services/pixel-agent/tests/test_workspace_preview_git.py index 4a65c488da..7c37773390 100644 --- a/ods/extensions/services/pixel-agent/tests/test_workspace_preview_git.py +++ b/ods/extensions/services/pixel-agent/tests/test_workspace_preview_git.py @@ -138,11 +138,14 @@ def test_exact_normal_repo_fixture(): canonical = _build_exact_repo_fixture(site) - # Pre-snapshot source tree snapshot for mutation check + # Compare the user's source files, not Git's own transient maintenance + # locks. Git may remove .git/objects/maintenance.lock after git commit + # returns, independently of publication. source_files_pre = {} for f in site.rglob("*"): - if f.is_file(): - source_files_pre[str(f.relative_to(site))] = f.read_bytes() + rel = f.relative_to(site) + if f.is_file() and ".git" not in rel.parts: + source_files_pre[str(rel)] = f.read_bytes() # Publish — must succeed with the revised patch result = MODULE.publish_snapshot(workspace, previews, "gitea-demo", os.getuid()) From 1ecd8d4f3a500307665e76dda6b6d42dccaba1d4 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 20:11:44 -0400 Subject: [PATCH 17/52] fix(pixel): block stale Lemonade model identity --- .../services/dashboard-api/routers/pixel.py | 51 ++++++++-- .../dashboard-api/tests/test_pixel.py | 98 +++++++++++++++++++ 2 files changed, 143 insertions(+), 6 deletions(-) diff --git a/ods/extensions/services/dashboard-api/routers/pixel.py b/ods/extensions/services/dashboard-api/routers/pixel.py index f58917ebd5..71989ba36c 100644 --- a/ods/extensions/services/dashboard-api/routers/pixel.py +++ b/ods/extensions/services/dashboard-api/routers/pixel.py @@ -27,6 +27,7 @@ from pixel_chat_results import ChatResultStore, ResultCapacity, ResultConflict, owner_namespace from security import verify_api_key from config import read_live_env_value +from helpers import get_loaded_model from pixel_chat_identity import asks_display_name, confirmed_display_name, display_name_stream, messages_with_identity from pixel_chat_context import HistorySnapshot, public_context @@ -63,6 +64,10 @@ ) _CONTROL = re.compile(r"[\x00-\x1f\x7f-\x9f]") _MODEL_SWITCH_DETAIL = "Model switch in progress; Pixel will be ready when activation completes" +_MODEL_IDENTITY_DETAIL = ( + "Pixel cannot verify its recorded model against the loaded Lemonade model. " + "Re-select the model in Models before using Pixel." +) _MODEL_ADAPTIVE_DETAIL = ( "Pixel is ready and adapts its tool flow for this model. Model capability " "affects the quality and persistence of complex work, not access or the " @@ -377,13 +382,13 @@ def _model_support_from_status(status: object) -> dict[str, str] | None: async def _model_readiness_issue() -> tuple[str, str] | None: - """Return a host-proven model transition, if present. + """Return a host-proven transition or an unverified Lemonade route. - A failed lifecycle probe does not falsely take down an otherwise healthy - Pixel edge. Model quality metadata is advisory; the edge readiness check - remains authoritative. + A failed host lifecycle probe alone does not take down the Pixel edge. + A recorded Lemonade route does require live identity proof before chat. + Model quality metadata remains advisory, not an access restriction. """ - return _model_readiness_issue_from_status(await _host_model_status()) + return await _model_readiness_issue_for_status(await _host_model_status()) def _active_runtime_projection(status: object) -> dict[str, object] | None: @@ -422,6 +427,40 @@ def _active_runtime_projection(status: object) -> dict[str, object] | None: return {key: runtime[key] for key in expected | {"routeFingerprint"} if key in runtime} +def _model_identity_tokens(value: str | None) -> set[str]: + """Compare a Lemonade ID with the equivalent GGUF basename, not a path.""" + if not isinstance(value, str) or not value.strip(): + return set() + name = Path(value.strip()).name.casefold() + tokens = {name} + if name.startswith("extra."): + tokens.add(name[6:]) + for token in tuple(tokens): + if token.endswith(".gguf"): + tokens.add(token[:-5]) + return tokens + + +async def _model_readiness_issue_for_status(status: object) -> tuple[str, str] | None: + issue = _model_readiness_issue_from_status(status) + if issue is not None: + return issue + runtime = _active_runtime_projection(status) + if (runtime is None or runtime.get("source") != "local-switchboard" + or read_live_env_value("LLM_BACKEND").strip().casefold() != "lemonade"): + return None + try: + loaded = await asyncio.wait_for(get_loaded_model(), timeout=3.0) + except Exception as exc: + # Probe failures cannot validate a recorded external route. Do not log + # exception text; it may contain the private backend origin or key. + logger.warning("Pixel Lemonade identity probe failed (%s)", type(exc).__name__) + return "model_unavailable", _MODEL_IDENTITY_DETAIL + if not (_model_identity_tokens(runtime["model"]) & _model_identity_tokens(loaded)): + return "model_unavailable", _MODEL_IDENTITY_DETAIL + return None + + async def _model_activation_in_progress() -> bool: """Compatibility wrapper retained for focused lifecycle callers/tests.""" issue = await _model_readiness_issue() @@ -446,7 +485,7 @@ async def pixel_status() -> dict[str, object]: if config is None: return {"available": False, "model": None, "detail": "Pixel is not enabled"} host_status = await _host_model_status() - readiness_issue = _model_readiness_issue_from_status(host_status) + readiness_issue = await _model_readiness_issue_for_status(host_status) if readiness_issue is not None: state, detail = readiness_issue return { diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel.py b/ods/extensions/services/dashboard-api/tests/test_pixel.py index 65741cd6b9..be764e1685 100644 --- a/ods/extensions/services/dashboard-api/tests/test_pixel.py +++ b/ods/extensions/services/dashboard-api/tests/test_pixel.py @@ -432,6 +432,104 @@ async def local_status(*_args, **_kwargs): assert result["modelSupport"]["tier"] == "adaptive" +@pytest.mark.asyncio +async def test_lemonade_model_drift_blocks_pixel_status_and_chat(monkeypatch): + runtime = {"source": "local-switchboard", "model": "Qwen3.6-35B-A3B-GGUF", + "contextLength": 65536} + + async def recorded_status(*_args, **_kwargs): + return {"status": "idle", "activeRuntime": runtime} + + async def physical_model(): + return "Qwen3.5-2B-Q4_K_M" + + monkeypatch.setattr(pixel, "request_agent_json", recorded_status) + monkeypatch.setattr(pixel, "read_live_env_value", + lambda key: "lemonade" if key == "LLM_BACKEND" else "") + monkeypatch.setattr(pixel, "get_loaded_model", physical_model) + with patch.object(pixel.httpx, "AsyncClient", + side_effect=AssertionError("stale route reached Pixel edge")): + status = await pixel.pixel_status() + body = pixel.ChatStreamRequest( + chat_id="drift-test", messages=[{"role": "user", "content": "hello"}] + ) + with pytest.raises(HTTPException) as raised: + await pixel.pixel_chat_stream(ConnectedRequest(), body) + assert status == { + "available": False, "model": None, "state": "model_unavailable", + "detail": pixel._MODEL_IDENTITY_DETAIL, + } + assert raised.value.status_code == 409 + assert raised.value.detail == pixel._MODEL_IDENTITY_DETAIL + + +@pytest.mark.asyncio +@pytest.mark.parametrize("loaded", ["Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-GGUF.gguf"]) +async def test_matching_lemonade_model_keeps_pixel_available(monkeypatch, loaded): + runtime = {"source": "local-switchboard", "model": "Qwen3.6-35B-A3B-GGUF", + "contextLength": 65536} + + async def recorded_status(*_args, **_kwargs): + return {"status": "idle", "activeRuntime": runtime} + + async def physical_model(): + return loaded + + monkeypatch.setattr(pixel, "request_agent_json", recorded_status) + monkeypatch.setattr(pixel, "read_live_env_value", + lambda key: "lemonade" if key == "LLM_BACKEND" else "") + monkeypatch.setattr(pixel, "get_loaded_model", physical_model) + body = json.dumps({"data": [{"id": "pixel/default"}]}).encode() + with patch.object(pixel.httpx, "AsyncClient", + return_value=FakeClient(FakeResponse(chunks=[body]))): + status = await pixel.pixel_status() + assert status["available"] is True + assert status["runtime"] == runtime + + +@pytest.mark.asyncio +async def test_lemonade_probe_failure_fails_closed_without_logging_endpoint(monkeypatch, caplog): + runtime = {"source": "local-switchboard", "model": "Qwen3.6-35B-A3B-GGUF", + "contextLength": 65536} + + async def recorded_status(*_args, **_kwargs): + return {"status": "idle", "activeRuntime": runtime} + + async def failed_probe(): + raise RuntimeError("private Lemonade origin and token") + + monkeypatch.setattr(pixel, "request_agent_json", recorded_status) + monkeypatch.setattr(pixel, "read_live_env_value", + lambda key: "lemonade" if key == "LLM_BACKEND" else "") + monkeypatch.setattr(pixel, "get_loaded_model", failed_probe) + result = await pixel.pixel_status() + assert result["available"] is False + assert result["state"] == "model_unavailable" + assert "private Lemonade" not in caplog.text + + +@pytest.mark.asyncio +async def test_non_lemonade_runtime_does_not_probe_lemonade_identity(monkeypatch): + runtime = {"source": "local-switchboard", "model": "local-model", + "contextLength": 65536} + + async def recorded_status(*_args, **_kwargs): + return {"status": "idle", "activeRuntime": runtime} + + async def forbidden_probe(): + raise AssertionError("non-Lemonade route was probed") + + monkeypatch.setattr(pixel, "request_agent_json", recorded_status) + monkeypatch.setattr(pixel, "read_live_env_value", lambda _key: "llama.cpp") + monkeypatch.setattr(pixel, "get_loaded_model", forbidden_probe) + body = json.dumps({"data": [{"id": "pixel/default"}]}).encode() + with patch.object(pixel.httpx, "AsyncClient", + return_value=FakeClient(FakeResponse(chunks=[body]))): + status = await pixel.pixel_status() + assert status["available"] is True + assert status["runtime"] == runtime + + def test_active_runtime_projection_accepts_a_constrained_adaptive_context(): runtime = { "source": "remote-provider", From a91d2986a4fd853d1787ae9b4a3e8bb16bd14c50 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 22:20:17 -0400 Subject: [PATCH 18/52] fix(installer): keep missing jq dry runs side-effect free --- ods/installers/lib/preflight-jq.sh | 29 ++++++++++++++ ods/installers/phases/01-preflight.sh | 18 +-------- .../contracts/test-installer-contracts.sh | 3 ++ ods/tests/test-installer-dry-run-jq.sh | 38 +++++++++++++++++++ ods/tests/test-podman-rootless-contracts.sh | 7 +++- 5 files changed, 77 insertions(+), 18 deletions(-) create mode 100644 ods/installers/lib/preflight-jq.sh create mode 100644 ods/tests/test-installer-dry-run-jq.sh diff --git a/ods/installers/lib/preflight-jq.sh b/ods/installers/lib/preflight-jq.sh new file mode 100644 index 0000000000..ed7066324d --- /dev/null +++ b/ods/installers/lib/preflight-jq.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# A dry run must never acquire missing host packages as a side effect. +ods_preflight_require_jq() { + if ! command -v jq >/dev/null 2>&1; then + if [[ "${DRY_RUN:-false}" == true ]]; then + error "jq is required for a complete dry run. Install jq yourself and retry; dry run will not install packages." + return 1 + fi + log "jq not found - attempting auto-install..." + if ! ods_sudo_available; then + error "jq is required but not installed and privileged package installation is unavailable. Install jq first, then re-run ODS." + return 1 + fi + case "$PKG_MANAGER" in + dnf) ods_sudo dnf install -y jq ;; + pacman) ods_sudo pacman -S --noconfirm jq ;; + zypper) ods_sudo zypper install -y jq ;; + apk) ods_sudo apk add jq ;; + apt) ods_sudo apt-get update -qq && ods_sudo apt-get install -y jq ;; + *) ods_sudo apt-get install -y jq ;; + esac + if ! command -v jq >/dev/null 2>&1; then + error "Failed to install jq automatically. Install it manually and re-run." + return 1 + fi + fi + log "jq: $(jq --version 2>/dev/null)" +} diff --git a/ods/installers/phases/01-preflight.sh b/ods/installers/phases/01-preflight.sh index 8b1c231cf2..bc5ad87c92 100755 --- a/ods/installers/phases/01-preflight.sh +++ b/ods/installers/phases/01-preflight.sh @@ -44,22 +44,8 @@ if ! command -v curl &> /dev/null; then fi log "curl: $(curl --version 2>/dev/null | sed -n '1p')" -if ! command -v jq &> /dev/null; then - log "jq not found - attempting auto-install..." - if ! ods_sudo_available; then - error "jq is required but not installed and privileged package installation is unavailable. Install jq first, then re-run ODS." - fi - case "$PKG_MANAGER" in - dnf) ods_sudo dnf install -y jq ;; - pacman) ods_sudo pacman -S --noconfirm jq ;; - zypper) ods_sudo zypper install -y jq ;; - apk) ods_sudo apk add jq ;; - apt) ods_sudo apt-get update -qq && ods_sudo apt-get install -y jq ;; - *) ods_sudo apt-get install -y jq ;; - esac - command -v jq &> /dev/null || error "Failed to install jq automatically. Install it manually and re-run." -fi -log "jq: $(jq --version 2>/dev/null)" +source "$SCRIPT_DIR/installers/lib/preflight-jq.sh" +ods_preflight_require_jq # Check optional tools (warn but don't fail) OPTIONAL_TOOLS_MISSING="" diff --git a/ods/tests/contracts/test-installer-contracts.sh b/ods/tests/contracts/test-installer-contracts.sh index afcaef7dc4..c5b8d374f3 100755 --- a/ods/tests/contracts/test-installer-contracts.sh +++ b/ods/tests/contracts/test-installer-contracts.sh @@ -559,6 +559,9 @@ for f in "${_resolver_callers[@]}"; do done unset _resolver_callers +echo "[contract] dry-run does not install a missing jq prerequisite" +bash tests/test-installer-dry-run-jq.sh + echo "[contract] optional extension compose files are installer-gated" bash tests/test-installer-feature-state-sync.sh # Bundled optional/recommended services that ship compose.yaml must not enter diff --git a/ods/tests/test-installer-dry-run-jq.sh b/ods/tests/test-installer-dry-run-jq.sh new file mode 100644 index 0000000000..c6dab3ef08 --- /dev/null +++ b/ods/tests/test-installer-dry-run-jq.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +source "$ROOT/installers/lib/preflight-jq.sh" + +tmp=$(mktemp -d) +cleanup() { + rm -f -- "$tmp/dry-run.out" "$tmp/install.out" "$tmp/privilege-calls" + rmdir -- "$tmp" +} +trap cleanup EXIT +log() { :; } +error() { printf '%s\n' "$*" >&2; return 1; } +ods_sudo_available() { printf 'privilege-check\n' >> "$tmp/privilege-calls"; return 0; } +ods_sudo() { printf '%s\n' "$*" >> "$tmp/privilege-calls"; return 0; } +command() { + if [[ "${1:-}" == -v && "${2:-}" == jq ]]; then return 1; fi + builtin command "$@" +} + +DRY_RUN=true PKG_MANAGER=apt +if ods_preflight_require_jq >"$tmp/dry-run.out" 2>&1; then + echo 'dry run accepted a missing required jq' >&2; exit 1 +fi +grep -Fq 'dry run will not install packages' "$tmp/dry-run.out" +if [[ -e "$tmp/privilege-calls" ]]; then + echo 'dry run attempted a privileged package operation' >&2; exit 1 +fi + +DRY_RUN=false +if ods_preflight_require_jq >"$tmp/install.out" 2>&1; then + echo 'real install accepted a still-missing jq' >&2; exit 1 +fi +grep -Fxq 'privilege-check' "$tmp/privilege-calls" +grep -Fxq 'apt-get update -qq' "$tmp/privilege-calls" +grep -Fxq 'apt-get install -y jq' "$tmp/privilege-calls" +printf '%s\n' 'installer dry-run jq prerequisite tests passed' diff --git a/ods/tests/test-podman-rootless-contracts.sh b/ods/tests/test-podman-rootless-contracts.sh index dd031d72d5..a4a872fa0f 100644 --- a/ods/tests/test-podman-rootless-contracts.sh +++ b/ods/tests/test-podman-rootless-contracts.sh @@ -267,11 +267,14 @@ grep -q 'not owned by ODS' "$agent_output" \ kill "$unowned_pid" 2>/dev/null || true pass "session fallback refuses to stop an unowned stale PID" -python3 - "$ROOT_DIR/installers/phases/01-preflight.sh" <<'PY' +python3 - "$ROOT_DIR/installers/phases/01-preflight.sh" "$ROOT_DIR/installers/lib/preflight-jq.sh" <<'PY' import pathlib import sys -text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +phase = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +assert 'source "$SCRIPT_DIR/installers/lib/preflight-jq.sh"' in phase +assert "ods_preflight_require_jq" in phase +text = pathlib.Path(sys.argv[2]).read_text(encoding="utf-8") start = text.index("if ! command -v jq") end = text.index('log "jq:', start) block = text[start:end] From eeab8e04723234d18e5ce6bb9128d8fb5807dc34 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 22:30:04 -0400 Subject: [PATCH 19/52] fix(pixel): route cloud and external relay through LiteLLM --- ods/docker-compose.cloud.yml | 8 ---- .../pixel-model-relay/compose.yaml.disabled | 6 +-- .../services/pixel-model-relay/relay.py | 30 ++++++++++--- .../pixel-model-relay/tests/test_relay.py | 43 ++++++++++++++++++ ods/installers/lib/pixel-host-install.sh | 6 +++ .../contracts/test-installer-contracts.sh | 1 + ods/tests/test-linux-cloud-mode.sh | 7 ++- ods/tests/test-pixel-model-relay-compose.sh | 45 +++++++++++++++++++ 8 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 ods/tests/test-pixel-model-relay-compose.sh diff --git a/ods/docker-compose.cloud.yml b/ods/docker-compose.cloud.yml index 1fe6f5b96f..6f4fc45469 100644 --- a/ods/docker-compose.cloud.yml +++ b/ods/docker-compose.cloud.yml @@ -16,11 +16,3 @@ services: profiles: - local-inference restart: "no" - - # Pixel's model relay has a hard dependency on model-router. Keep their - # cloud-mode profile in lockstep so an enabled local install remains a valid - # Compose project when the operator checks or switches to cloud mode. - pixel-model-relay: - profiles: - - local-inference - restart: "no" diff --git a/ods/extensions/services/pixel-model-relay/compose.yaml.disabled b/ods/extensions/services/pixel-model-relay/compose.yaml.disabled index 236379bb0b..d81c6d0288 100644 --- a/ods/extensions/services/pixel-model-relay/compose.yaml.disabled +++ b/ods/extensions/services/pixel-model-relay/compose.yaml.disabled @@ -12,11 +12,11 @@ services: security_opt: [no-new-privileges:true] environment: - PIXEL_MODEL_RELAY_KEY=${PIXEL_MODEL_RELAY_KEY:?Set PIXEL_MODEL_RELAY_KEY in .env} + - ODS_MODE=${ODS_MODE:-local} + - EXTERNAL_LLM_URL=${EXTERNAL_LLM_URL:-} + - LITELLM_KEY=${LITELLM_KEY:-} ports: - "127.0.0.1:${PIXEL_MODEL_RELAY_PORT:-4006}:4102" - depends_on: - model-router: - condition: service_healthy healthcheck: test: [CMD, python3, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4102/health', timeout=4)"] interval: 15s diff --git a/ods/extensions/services/pixel-model-relay/relay.py b/ods/extensions/services/pixel-model-relay/relay.py index aaf1ba1899..4932558672 100644 --- a/ods/extensions/services/pixel-model-relay/relay.py +++ b/ods/extensions/services/pixel-model-relay/relay.py @@ -1,8 +1,8 @@ -"""Authenticated, Pixel-only host loopback bridge to the ODS model router. +"""Authenticated, Pixel-only host loopback bridge to the ODS model route. -The router remains the single dynamic model/swap/telemetry authority. This -bridge exists because the shared LiteLLM proxy can retain inference after an -OpenClaw client disconnects. It cannot forward arbitrary URLs or endpoints. +Managed inference uses model-router as the dynamic model/swap authority. +Cloud and external-LLM installs have no managed router, so their fixed route +is the authenticated LiteLLM gateway. No caller can select a URL or endpoint. """ import asyncio @@ -14,7 +14,17 @@ from aiohttp import ClientSession, ClientTimeout, web KEY = os.environ.get("PIXEL_MODEL_RELAY_KEY", "") -UPSTREAM = "http://model-router:9099" +LITELLM_KEY = os.environ.get("LITELLM_KEY", "") + + +def _upstream_route(ods_mode, external_llm_url): + if ods_mode == "cloud" or external_llm_url: + return "http://litellm:4000", True + return "http://model-router:9099", False + + +UPSTREAM, UPSTREAM_REQUIRES_KEY = _upstream_route( + os.environ.get("ODS_MODE", "local"), os.environ.get("EXTERNAL_LLM_URL", "")) ALIASES = {"ods/current", "default"} MAX_BODY = 2 * 1024 * 1024 WRITE_TIMEOUT_SECONDS = 30.0 # Host-local OpenClaw must drain promptly. @@ -50,9 +60,12 @@ async def _inference(request): raise web.HTTPBadRequest() async with ClientSession(timeout=ClientTimeout(total=None)) as client: + upstream_headers = {"Content-Type": "application/json"} + if UPSTREAM_REQUIRES_KEY: + upstream_headers["Authorization"] = "Bearer " + LITELLM_KEY upstream_task = asyncio.create_task(client.request( request.method, UPSTREAM + request.path, data=body, - headers={"Content-Type": "application/json"})) + headers=upstream_headers)) disconnected = asyncio.create_task(_disconnect(request)) try: done, _ = await asyncio.wait({upstream_task, disconnected}, return_when=asyncio.FIRST_COMPLETED) @@ -104,6 +117,11 @@ def create_app(): if not KEY or not KEY.isascii() or len(KEY) > 4096 \ or any(ord(c) < 32 or ord(c) == 127 for c in KEY): raise RuntimeError("invalid Pixel model relay key") + if UPSTREAM_REQUIRES_KEY and ( + not LITELLM_KEY or not LITELLM_KEY.isascii() or len(LITELLM_KEY) > 4096 + or any(ord(c) < 32 or ord(c) == 127 for c in LITELLM_KEY) + ): + raise RuntimeError("invalid LiteLLM model relay key") app = web.Application(client_max_size=MAX_BODY) app.router.add_get("/health", _health) app.router.add_route("*", "/v1/models", _inference) diff --git a/ods/extensions/services/pixel-model-relay/tests/test_relay.py b/ods/extensions/services/pixel-model-relay/tests/test_relay.py index fd54f7b2b5..9506a83a5e 100644 --- a/ods/extensions/services/pixel-model-relay/tests/test_relay.py +++ b/ods/extensions/services/pixel-model-relay/tests/test_relay.py @@ -9,6 +9,8 @@ from aiohttp import ClientSession, web os.environ["PIXEL_MODEL_RELAY_KEY"] = "test-only-pixel-relay-key" +os.environ["ODS_MODE"] = "local" +os.environ["EXTERNAL_LLM_URL"] = "" spec = importlib.util.spec_from_file_location("relay", Path(__file__).parents[1] / "relay.py") relay = importlib.util.module_from_spec(spec) spec.loader.exec_module(relay) @@ -101,6 +103,47 @@ async def test_non_ascii_key_fails_at_startup(self): finally: relay.KEY = original + async def test_cloud_and_external_routes_are_fixed_internal_targets(self): + self.assertEqual(relay._upstream_route("local", ""), + ("http://model-router:9099", False)) + self.assertEqual(relay._upstream_route("cloud", ""), + ("http://litellm:4000", True)) + self.assertEqual(relay._upstream_route("local", "http://untrusted.example/v1"), + ("http://litellm:4000", True)) + + async def test_litellm_route_uses_only_its_gateway_key(self): + seen = [] + + async def keyed_models(request): + seen.append(request.headers.get("Authorization")) + return web.json_response({"data": [{"id": "ods/current"}]}) + + keyed = web.Application() + keyed.router.add_get("/v1/models", keyed_models) + runner, upstream = await start(keyed) + prior = relay.UPSTREAM, relay.UPSTREAM_REQUIRES_KEY, relay.LITELLM_KEY + relay.UPSTREAM, relay.UPSTREAM_REQUIRES_KEY, relay.LITELLM_KEY = ( + upstream, True, "litellm-only-test-key") + try: + async with ClientSession() as client: + async with client.get(self.url + "/v1/models", headers={ + "Authorization": "Bearer test-only-pixel-relay-key" + }) as response: + self.assertEqual(response.status, 200) + self.assertEqual(seen, ["Bearer litellm-only-test-key"]) + finally: + relay.UPSTREAM, relay.UPSTREAM_REQUIRES_KEY, relay.LITELLM_KEY = prior + await runner.cleanup() + + async def test_litellm_route_requires_a_valid_gateway_key(self): + prior = relay.UPSTREAM_REQUIRES_KEY, relay.LITELLM_KEY + relay.UPSTREAM_REQUIRES_KEY, relay.LITELLM_KEY = True, "" + try: + with self.assertRaisesRegex(RuntimeError, "invalid LiteLLM model relay key"): + relay.create_app() + finally: + relay.UPSTREAM_REQUIRES_KEY, relay.LITELLM_KEY = prior + if __name__ == "__main__": unittest.main() diff --git a/ods/installers/lib/pixel-host-install.sh b/ods/installers/lib/pixel-host-install.sh index 156a503a43..6467da3fe0 100644 --- a/ods/installers/lib/pixel-host-install.sh +++ b/ods/installers/lib/pixel-host-install.sh @@ -4601,6 +4601,12 @@ ods_pixel_install_default_agent() { # provider is Parallel. ODS also shares SearXNG with OWUI/Perplexica, so # a clean install must start it before Pixel plans its host deployment. local -a pixel_prerequisites=(litellm dashboard-api pixel-edge pixel-model-relay searxng) + # Managed inference needs the router before the relay's real model probe. + # Cloud/external installs instead bind the relay to authenticated LiteLLM; + # their Compose overlays intentionally profile model-router out. + if [[ "${ODS_MODE:-local}" != cloud && -z "${EXTERNAL_LLM_URL:-}" ]]; then + pixel_prerequisites+=(model-router) + fi owner="${PIXEL_SERVICE_USER:-$(ods_pixel_install_owner)}" || return 1 home="$(ods_pixel_owner_home "$owner")" || return 1 pixel_gateway_port="$(_ods_pixel_gateway_port)" || { diff --git a/ods/tests/contracts/test-installer-contracts.sh b/ods/tests/contracts/test-installer-contracts.sh index c5b8d374f3..7a01ed9135 100755 --- a/ods/tests/contracts/test-installer-contracts.sh +++ b/ods/tests/contracts/test-installer-contracts.sh @@ -599,6 +599,7 @@ done echo "[contract] SearXNG follows web search consumers, not only --recommended" bash tests/test-pixel-support-services.sh +bash tests/test-pixel-model-relay-compose.sh grep -qE 'ENABLE_RECOMMENDED:-false' "$features_phase" \ || { echo "[FAIL] ENABLE_SEARXNG derivation must consult ENABLE_RECOMMENDED"; exit 1; } grep -qE 'ENABLE_PIXEL_RUNTIME:-false' "$features_phase" \ diff --git a/ods/tests/test-linux-cloud-mode.sh b/ods/tests/test-linux-cloud-mode.sh index 89c20e5983..4b20d435d4 100644 --- a/ods/tests/test-linux-cloud-mode.sh +++ b/ods/tests/test-linux-cloud-mode.sh @@ -111,12 +111,15 @@ import sys import yaml services = yaml.safe_load(Path("docker-compose.cloud.yml").read_text(encoding="utf-8"))["services"] -for name in ("llama-server", "model-router", "pixel-model-relay"): +for name in ("llama-server", "model-router"): service = services.get(name, {}) if "local-inference" not in service.get("profiles", []) or service.get("restart") != "no": print(f"[FAIL] cloud mode must profile {name} out with its local dependency chain", file=sys.stderr) sys.exit(1) -print("[PASS] cloud mode profiles Pixel relay and its local model dependency together") +if "pixel-model-relay" in services: + print("[FAIL] cloud overlay must not disable the enabled Pixel relay", file=sys.stderr) + sys.exit(1) +print("[PASS] cloud mode profiles local inference out and retains Pixel's external gateway route") PY if grep -Fq -- '--ods-mode "${ODS_MODE:-local}"' installers/lib/compose-select.sh \ diff --git a/ods/tests/test-pixel-model-relay-compose.sh b/ods/tests/test-pixel-model-relay-compose.sh new file mode 100644 index 0000000000..17739505b9 --- /dev/null +++ b/ods/tests/test-pixel-model-relay-compose.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$root" +if ! command -v docker >/dev/null 2>&1 || ! docker compose version >/dev/null 2>&1; then + echo '[SKIP] Docker Compose is unavailable for Pixel relay routing matrix' + exit 0 +fi + +for mode in managed cloud external; do + flags=( + -f docker-compose.base.yml + -f docker-compose.cpu.yml + -f extensions/services/litellm/compose.yaml + -f extensions/services/pixel-model-relay/compose.yaml.disabled + ) + ods_mode=local external_url= + case "$mode" in + managed) ;; + cloud) ods_mode=cloud; flags+=(-f docker-compose.cloud.yml) ;; + external) + external_url=http://host.docker.internal:8080 + flags+=(-f docker-compose.external-llm.yml) + ;; + esac + rendered="$(PIXEL_MODEL_RELAY_KEY=test-relay-key LITELLM_KEY=test-litellm-key \ + ODS_MODE="$ods_mode" EXTERNAL_LLM_URL="$external_url" \ + docker compose --env-file .env.example "${flags[@]}" config --format json)" + printf '%s\n' "$rendered" | python3 -c ' +import json +import sys + +mode = sys.argv[1] +services = json.load(sys.stdin)["services"] +assert "pixel-model-relay" in services +assert "litellm" in services +assert ("model-router" in services) == (mode == "managed") +relay_env = services["pixel-model-relay"]["environment"] +assert relay_env["ODS_MODE"] == ("cloud" if mode == "cloud" else "local") +assert bool(relay_env["EXTERNAL_LLM_URL"]) == (mode == "external") +assert relay_env["LITELLM_KEY"] == "test-litellm-key" +' "$mode" +done +echo 'Pixel relay Compose routes pass managed, cloud, and external modes' From e61932183e291afa9258c30672738bb8708d8725 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 23:02:59 -0400 Subject: [PATCH 20/52] fix(n8n): support nonstandard host UIDs on Linux --- ods/FAQ.md | 2 +- ods/extensions/services/n8n/README.md | 4 ++-- ods/extensions/services/n8n/compose.yaml | 9 +++++++-- ods/tests/contracts/test-installer-contracts.sh | 3 +++ ods/tests/test-n8n-cookie-policy.sh | 12 ++++++++++++ 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/ods/FAQ.md b/ods/FAQ.md index 4682673e56..93a05a1dab 100644 --- a/ods/FAQ.md +++ b/ods/FAQ.md @@ -485,7 +485,7 @@ SQLite databases are in Docker volumes: Access via: ```bash -docker compose exec n8n sqlite3 /home/node/.n8n/database.sqlite +docker compose exec n8n sqlite3 /tmp/.n8n/database.sqlite ``` ### Can I use OpenAI/Anthropic APIs? diff --git a/ods/extensions/services/n8n/README.md b/ods/extensions/services/n8n/README.md index 299a51d196..7ed4576e71 100644 --- a/ods/extensions/services/n8n/README.md +++ b/ods/extensions/services/n8n/README.md @@ -66,8 +66,8 @@ curl -X POST http://localhost:3002/api/workflows/my-workflow-id/enable | Path (host) | Mounted at (container) | Contents | |-------------|------------------------|----------| -| `data/n8n/` | `/home/node/.n8n` | Workflows, credentials, execution history | -| `config/n8n/` | `/home/node/workflows` | Pre-built workflow templates | +| `data/n8n/` | `/tmp/.n8n` | Workflows, credentials, execution history (persistent bind mount) | +| `config/n8n/` | `/tmp/workflows` | Pre-built workflow templates | ## LLM Integration diff --git a/ods/extensions/services/n8n/compose.yaml b/ods/extensions/services/n8n/compose.yaml index 912a778ef4..9a4a397653 100644 --- a/ods/extensions/services/n8n/compose.yaml +++ b/ods/extensions/services/n8n/compose.yaml @@ -8,6 +8,11 @@ services: - no-new-privileges:true entrypoint: ["tini", "--", "/bin/sh", "/opt/ods/n8n-entrypoint.sh"] environment: + # A host UID may not exist in the image passwd database (for example, + # Colima's 501). The image's /home/node is not traversable by that UID; + # keep the persistent data mount at n8n's resolved user-folder path. + - HOME=/tmp + - N8N_USER_FOLDER=/tmp - N8N_DEFAULT_ADMIN_EMAIL=${N8N_USER:?N8N_USER must be set in .env} - N8N_DEFAULT_ADMIN_PASSWORD=${N8N_PASS:?N8N_PASS must be set in .env} - N8N_HOST=${N8N_HOST:-localhost} @@ -21,8 +26,8 @@ services: volumes: - ./extensions/services/n8n/n8n-cookie-policy.sh:/opt/ods/n8n-cookie-policy.sh:ro,z - ./extensions/services/n8n/n8n-entrypoint.sh:/opt/ods/n8n-entrypoint.sh:ro,z - - ./data/n8n:/home/node/.n8n:z - - ./config/n8n:/home/node/workflows:z + - ./data/n8n:/tmp/.n8n:z + - ./config/n8n:/tmp/workflows:z ports: - "${BIND_ADDRESS:-127.0.0.1}:${N8N_PORT:-5678}:5678" deploy: diff --git a/ods/tests/contracts/test-installer-contracts.sh b/ods/tests/contracts/test-installer-contracts.sh index 7a01ed9135..e3fbe93443 100755 --- a/ods/tests/contracts/test-installer-contracts.sh +++ b/ods/tests/contracts/test-installer-contracts.sh @@ -9,6 +9,9 @@ command -v jq >/dev/null 2>&1 || { exit 1 } +echo "[contract] n8n nonstandard-UID home and cookie policy" +bash tests/test-n8n-cookie-policy.sh + echo "[contract] backend contract files" for f in config/backends/amd.json config/backends/nvidia.json config/backends/cpu.json config/backends/apple.json; do test -f "$f" || { echo "[FAIL] missing $f"; exit 1; } diff --git a/ods/tests/test-n8n-cookie-policy.sh b/ods/tests/test-n8n-cookie-policy.sh index cf55874861..cd7bc152a1 100755 --- a/ods/tests/test-n8n-cookie-policy.sh +++ b/ods/tests/test-n8n-cookie-policy.sh @@ -47,6 +47,18 @@ if ! grep -Fq "$expected_entrypoint" "$compose_file"; then printf 'FAIL n8n entrypoint must run through /bin/sh for Windows bind mounts\n' >&2 failures=$((failures + 1)) fi +for required_setting in 'HOME=/tmp' 'N8N_USER_FOLDER=/tmp'; do + if ! grep -Fq -- "- $required_setting" "$compose_file"; then + printf 'FAIL n8n must set %s for arbitrary host UIDs\n' "$required_setting" >&2 + failures=$((failures + 1)) + fi +done +for required_mount in './data/n8n:/tmp/.n8n:z' './config/n8n:/tmp/workflows:z'; do + if ! grep -Fq -- "- $required_mount" "$compose_file"; then + printf 'FAIL n8n must mount %s outside the image-owned home\n' "$required_mount" >&2 + failures=$((failures + 1)) + fi +done if ((failures > 0)); then printf '%d n8n cookie policy test(s) failed\n' "$failures" >&2 From 2d756a5f81bf62c8bab99472829ddb6c788d66d3 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 23:11:52 -0400 Subject: [PATCH 21/52] fix(preflight): check LiteLLM for external model installs --- ods/lib/preflight-llm-route.sh | 14 +++++++ ods/ods-preflight.sh | 13 ++----- ods/scripts/ods-preflight.sh | 23 ++++++++--- .../contracts/test-installer-contracts.sh | 3 ++ ods/tests/test-ods-preflight-llm-route.sh | 39 +++++++++++++++++++ 5 files changed, 77 insertions(+), 15 deletions(-) create mode 100644 ods/lib/preflight-llm-route.sh create mode 100644 ods/tests/test-ods-preflight-llm-route.sh diff --git a/ods/lib/preflight-llm-route.sh b/ods/lib/preflight-llm-route.sh new file mode 100644 index 0000000000..eb9477571f --- /dev/null +++ b/ods/lib/preflight-llm-route.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Select the endpoint that serves the installed model, not the detected GPU. + +is_external_lemonade() { + local external="${LEMONADE_EXTERNAL:-false}" + local managed="${AMD_INFERENCE_MANAGED:-}" + local mode="${ODS_MODE:-local}" + [[ "${external,,}" == "true" ]] || [[ "${mode,,}" == "lemonade" && "${managed,,}" == "false" ]] +} + +ods_preflight_uses_litellm() { + local mode="${ODS_MODE:-local}" + is_external_lemonade || [[ -n "${EXTERNAL_LLM_URL:-}" ]] || [[ "${mode,,}" == "cloud" ]] +} diff --git a/ods/ods-preflight.sh b/ods/ods-preflight.sh index ba3fee0681..3a2fa2c7b5 100755 --- a/ods/ods-preflight.sh +++ b/ods/ods-preflight.sh @@ -21,6 +21,8 @@ LOG_FILE="$ODS_DIR/preflight-$(date +%Y%m%d-%H%M%S).log" # Safe .env loading (no eval; use lib/safe-env.sh) [[ -f "$ODS_DIR/lib/safe-env.sh" ]] && . "$ODS_DIR/lib/safe-env.sh" load_env_file "$ODS_DIR/.env" +# shellcheck source=lib/preflight-llm-route.sh +. "$ODS_DIR/lib/preflight-llm-route.sh" SERVICE_HOST="${SERVICE_HOST:-localhost}" @@ -72,13 +74,6 @@ detect_backend() { BACKEND=$(detect_backend) -is_external_lemonade() { - local external="${LEMONADE_EXTERNAL:-false}" - local managed="${AMD_INFERENCE_MANAGED:-}" - local mode="${ODS_MODE:-local}" - [[ "${external,,}" == "true" ]] || [[ "${mode,,}" == "lemonade" && "${managed,,}" == "false" ]] -} - # Colors RED='\033[0;31m' GREEN='\033[0;32m' @@ -196,10 +191,10 @@ log "" # OLLAMA_PORT=11434 to .env automatically — it will be picked up via the # ${OLLAMA_PORT:-...} expansion below, so the fallback should be 8080. log "[4/8] Checking LLM endpoint..." -if is_external_lemonade; then +if ods_preflight_uses_litellm; then LLM_PORT="${LITELLM_PORT:-4000}" LLM_ENDPOINTS=("http://${SERVICE_HOST}:${LLM_PORT}/health/readiness" "http://127.0.0.1:${LLM_PORT}/health/readiness" "http://127.0.0.1:${LLM_PORT}/v1/models") - LLM_SERVICE_NAME="LiteLLM external Lemonade gateway" + LLM_SERVICE_NAME="LiteLLM gateway" LLM_CONTAINER_MATCH="ods-litellm" LLM_START_CMD="docker compose up -d litellm" else diff --git a/ods/scripts/ods-preflight.sh b/ods/scripts/ods-preflight.sh index 2c76671196..b026622fb7 100755 --- a/ods/scripts/ods-preflight.sh +++ b/ods/scripts/ods-preflight.sh @@ -14,6 +14,8 @@ sr_load # Safe .env loading for port overrides (no eval; use lib/safe-env.sh) [[ -f "$SCRIPT_DIR/lib/safe-env.sh" ]] && . "$SCRIPT_DIR/lib/safe-env.sh" load_env_file "$SCRIPT_DIR/.env" +# shellcheck source=../lib/preflight-llm-route.sh +. "$SCRIPT_DIR/lib/preflight-llm-route.sh" sr_resolve_ports # Resolve compose flags for accurate status checks @@ -41,6 +43,13 @@ echo "" LLM_PORT="${OLLAMA_PORT:-${LLAMA_SERVER_PORT:-${SERVICE_PORTS[llama-server]:-11434}}}" LLM_HEALTH="${SERVICE_HEALTH[llama-server]:-/health}" LLM_CONTAINER="${SERVICE_CONTAINERS[llama-server]:-ods-llama-server}" +LLM_NAME="llama-server" +if ods_preflight_uses_litellm; then + LLM_PORT="${LITELLM_PORT:-4000}" + LLM_HEALTH="/health/readiness" + LLM_CONTAINER="ods-litellm" + LLM_NAME="LiteLLM gateway" +fi WEBUI_PORT="${SERVICE_PORTS[open-webui]:-3000}" WEBUI_HEALTH="${SERVICE_HEALTH[open-webui]:-/}" @@ -64,16 +73,16 @@ else exit 1 fi -# Check llama-server health +# Check the selected model gateway health CURL_HEALTH_FLAGS=(--connect-timeout 3 --max-time 10) -echo -n "llama-server API (port $LLM_PORT)... " +echo -n "$LLM_NAME API (port $LLM_PORT)... " if curl -sf "${CURL_HEALTH_FLAGS[@]}" "http://127.0.0.1:${LLM_PORT}${LLM_HEALTH}" >/dev/null 2>&1; then echo -e "${GREEN}✓ healthy${NC}" else echo -e "${YELLOW}⚠ starting up${NC}" - echo " The model is still loading. Wait 1-2 minutes and retry." - echo " Monitor: docker compose logs -f llama-server" + echo " The model gateway is not ready. Wait and retry." + echo " Monitor: docker logs $LLM_CONTAINER" fi # Check WebUI @@ -84,9 +93,11 @@ else echo -e "${YELLOW}⚠ not ready${NC}" fi -# Check GPU if available +# Check GPU only when ODS owns a local inference container. echo -n "GPU availability... " -if docker exec "$LLM_CONTAINER" nvidia-smi >/dev/null 2>&1; then +if ods_preflight_uses_litellm; then + echo -e "${YELLOW}⚠ external model route (host GPU not required)${NC}" +elif docker exec "$LLM_CONTAINER" nvidia-smi >/dev/null 2>&1; then GPU_MEM=$(docker exec "$LLM_CONTAINER" nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits 2>/dev/null | sed -n '1p' | tr -d ' ') echo -e "${GREEN}✓ detected (${GPU_MEM}MB free)${NC}" else diff --git a/ods/tests/contracts/test-installer-contracts.sh b/ods/tests/contracts/test-installer-contracts.sh index e3fbe93443..617c619099 100755 --- a/ods/tests/contracts/test-installer-contracts.sh +++ b/ods/tests/contracts/test-installer-contracts.sh @@ -12,6 +12,9 @@ command -v jq >/dev/null 2>&1 || { echo "[contract] n8n nonstandard-UID home and cookie policy" bash tests/test-n8n-cookie-policy.sh +echo "[contract] installed preflight model route" +bash tests/test-ods-preflight-llm-route.sh + echo "[contract] backend contract files" for f in config/backends/amd.json config/backends/nvidia.json config/backends/cpu.json config/backends/apple.json; do test -f "$f" || { echo "[FAIL] missing $f"; exit 1; } diff --git a/ods/tests/test-ods-preflight-llm-route.sh b/ods/tests/test-ods-preflight-llm-route.sh new file mode 100644 index 0000000000..0f9e20059d --- /dev/null +++ b/ods/tests/test-ods-preflight-llm-route.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=../lib/preflight-llm-route.sh +source "$ROOT_DIR/lib/preflight-llm-route.sh" + +assert_route() { + local expected="$1" mode="$2" url="$3" external="$4" managed="$5" actual=local + if ( + export ODS_MODE="$mode" EXTERNAL_LLM_URL="$url" + export LEMONADE_EXTERNAL="$external" AMD_INFERENCE_MANAGED="$managed" + ods_preflight_uses_litellm + ); then actual=litellm; fi + if [[ "$actual" != "$expected" ]]; then + printf 'FAIL expected %s route, got %s\n' "$expected" "$actual" >&2 + exit 1 + fi +} + +assert_route local local '' false true +assert_route litellm local http://model.example:8080 false true +assert_route litellm cloud '' false true +assert_route litellm lemonade '' false false +assert_route litellm local '' true true + +grep -Fq 'if ods_preflight_uses_litellm; then' "$ROOT_DIR/ods-preflight.sh" || { + printf 'FAIL preflight did not use the shared route selector\n' >&2 + exit 1 +} +grep -Fq 'if ods_preflight_uses_litellm; then' "$ROOT_DIR/scripts/ods-preflight.sh" || { + printf 'FAIL quick preflight did not use the shared route selector\n' >&2 + exit 1 +} +grep -Fq 'LLM_CONTAINER="ods-litellm"' "$ROOT_DIR/scripts/ods-preflight.sh" || { + printf 'FAIL quick preflight did not check the external model gateway\n' >&2 + exit 1 +} +printf 'ODS preflight LLM route tests passed\n' From 2f2a5393a4c44f154ac80a535f4b9daf2f2bcaae Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 23:13:45 -0400 Subject: [PATCH 22/52] test(preflight): follow shared LiteLLM route selector --- ods/tests/contracts/test-external-lemonade-contracts.sh | 8 +++++--- ods/tests/test-preflight.sh | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ods/tests/contracts/test-external-lemonade-contracts.sh b/ods/tests/contracts/test-external-lemonade-contracts.sh index db6fecb79c..8593b2c340 100644 --- a/ods/tests/contracts/test-external-lemonade-contracts.sh +++ b/ods/tests/contracts/test-external-lemonade-contracts.sh @@ -174,10 +174,12 @@ rm -f -- "$LOG_FILE" unset -f curl echo "[contract] external Lemonade preflight checks LiteLLM instead of managed llama-server" -grep -q 'is_external_lemonade()' ods-preflight.sh \ +grep -q 'is_external_lemonade()' lib/preflight-llm-route.sh \ || { echo "[FAIL] ods-preflight must detect external Lemonade mode"; exit 1; } -grep -q 'LiteLLM external Lemonade gateway' ods-preflight.sh \ - || { echo "[FAIL] ods-preflight must label the external Lemonade LiteLLM route"; exit 1; } +grep -q 'if ods_preflight_uses_litellm; then' ods-preflight.sh \ + || { echo "[FAIL] ods-preflight must select the external Lemonade LiteLLM route"; exit 1; } +grep -q 'LiteLLM gateway' ods-preflight.sh \ + || { echo "[FAIL] ods-preflight must label the LiteLLM route"; exit 1; } grep -q 'ods-litellm' ods-preflight.sh \ || { echo "[FAIL] ods-preflight must check ods-litellm for external Lemonade"; exit 1; } diff --git a/ods/tests/test-preflight.sh b/ods/tests/test-preflight.sh index f9e2eed4b4..f8a04e6222 100644 --- a/ods/tests/test-preflight.sh +++ b/ods/tests/test-preflight.sh @@ -121,8 +121,9 @@ else fi # 11. External Lemonade mode checks LiteLLM, not a managed llama-server container -if grep -q 'is_external_lemonade()' "$PREFLIGHT" \ - && grep -q 'LiteLLM external Lemonade gateway' "$PREFLIGHT" \ +if grep -q 'is_external_lemonade()' "$SCRIPT_DIR/../lib/preflight-llm-route.sh" \ + && grep -q 'if ods_preflight_uses_litellm; then' "$PREFLIGHT" \ + && grep -q 'LiteLLM gateway' "$PREFLIGHT" \ && grep -q 'ods-litellm' "$PREFLIGHT"; then pass "External Lemonade preflight checks LiteLLM gateway" else From db7f6e0aaf1e2ae90ae70211f37d76c1ab63f4e0 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 23:19:52 -0400 Subject: [PATCH 23/52] fix(status): omit disabled local model in external installs --- ods/ods-cli | 10 ++++++++++ ods/tests/test-ods-preflight-llm-route.sh | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/ods/ods-cli b/ods/ods-cli index 3e565b03e5..9504325517 100755 --- a/ods/ods-cli +++ b/ods/ods-cli @@ -24,6 +24,8 @@ while [[ -L "$_source" ]]; do [[ "$_source" != /* ]] && _source="$_dir/$_source" done SCRIPT_DIR="$(cd "$(dirname "$_source")" && pwd)" +# shellcheck source=lib/preflight-llm-route.sh +. "$SCRIPT_DIR/lib/preflight-llm-route.sh" _resolve_cli_install_dir() { if [[ -n "${INSTALL_DIR:-}" ]]; then @@ -1355,6 +1357,11 @@ cmd_status() { # Launch parallel health checks for sid in "${SERVICE_IDS[@]}"; do + # Cloud and external-model installs intentionally do not start the + # managed llama-server. Report the active LiteLLM gateway instead. + if [[ "$sid" == "llama-server" ]] && ods_preflight_uses_litellm; then + continue + fi local health="${SERVICE_HEALTH[$sid]}" local port_env="${SERVICE_PORT_ENVS[$sid]}" local default_port="${SERVICE_PORTS[$sid]}" @@ -1462,6 +1469,9 @@ cmd_status_json() { trap 'rm -f "$tmp"' RETURN for sid in "${SERVICE_IDS[@]}"; do + if [[ "$sid" == "llama-server" ]] && ods_preflight_uses_litellm; then + continue + fi local cat="${SERVICE_CATEGORIES[$sid]}" local health="${SERVICE_HEALTH[$sid]}" local port_env="${SERVICE_PORT_ENVS[$sid]}" diff --git a/ods/tests/test-ods-preflight-llm-route.sh b/ods/tests/test-ods-preflight-llm-route.sh index 0f9e20059d..204bef8f06 100644 --- a/ods/tests/test-ods-preflight-llm-route.sh +++ b/ods/tests/test-ods-preflight-llm-route.sh @@ -36,4 +36,8 @@ grep -Fq 'LLM_CONTAINER="ods-litellm"' "$ROOT_DIR/scripts/ods-preflight.sh" || { printf 'FAIL quick preflight did not check the external model gateway\n' >&2 exit 1 } +if [[ "$(grep -Fc '[[ "$sid" == "llama-server" ]] && ods_preflight_uses_litellm' "$ROOT_DIR/ods-cli")" != 2 ]]; then + printf 'FAIL text and JSON status must omit disabled managed inference\n' >&2 + exit 1 +fi printf 'ODS preflight LLM route tests passed\n' From cab782c66cf3ffbee9c0ba61d480a912351f144e Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 23:22:07 -0400 Subject: [PATCH 24/52] fix(status): omit disabled model router in external installs --- ods/ods-cli | 6 +++--- ods/tests/test-ods-preflight-llm-route.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ods/ods-cli b/ods/ods-cli index 9504325517..545d7b2dc5 100755 --- a/ods/ods-cli +++ b/ods/ods-cli @@ -1358,8 +1358,8 @@ cmd_status() { # Launch parallel health checks for sid in "${SERVICE_IDS[@]}"; do # Cloud and external-model installs intentionally do not start the - # managed llama-server. Report the active LiteLLM gateway instead. - if [[ "$sid" == "llama-server" ]] && ods_preflight_uses_litellm; then + # managed model stack. Report the active LiteLLM gateway instead. + if [[ "$sid" == "llama-server" || "$sid" == "model-router" ]] && ods_preflight_uses_litellm; then continue fi local health="${SERVICE_HEALTH[$sid]}" @@ -1469,7 +1469,7 @@ cmd_status_json() { trap 'rm -f "$tmp"' RETURN for sid in "${SERVICE_IDS[@]}"; do - if [[ "$sid" == "llama-server" ]] && ods_preflight_uses_litellm; then + if [[ "$sid" == "llama-server" || "$sid" == "model-router" ]] && ods_preflight_uses_litellm; then continue fi local cat="${SERVICE_CATEGORIES[$sid]}" diff --git a/ods/tests/test-ods-preflight-llm-route.sh b/ods/tests/test-ods-preflight-llm-route.sh index 204bef8f06..dccd31951f 100644 --- a/ods/tests/test-ods-preflight-llm-route.sh +++ b/ods/tests/test-ods-preflight-llm-route.sh @@ -36,7 +36,7 @@ grep -Fq 'LLM_CONTAINER="ods-litellm"' "$ROOT_DIR/scripts/ods-preflight.sh" || { printf 'FAIL quick preflight did not check the external model gateway\n' >&2 exit 1 } -if [[ "$(grep -Fc '[[ "$sid" == "llama-server" ]] && ods_preflight_uses_litellm' "$ROOT_DIR/ods-cli")" != 2 ]]; then +if [[ "$(grep -Fc '[[ "$sid" == "llama-server" || "$sid" == "model-router" ]] && ods_preflight_uses_litellm' "$ROOT_DIR/ods-cli")" != 2 ]]; then printf 'FAIL text and JSON status must omit disabled managed inference\n' >&2 exit 1 fi From 14079d573b94497965d3e610e04a03323e374ac1 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Thu, 17 Sep 2026 23:36:10 -0400 Subject: [PATCH 25/52] fix(cli): load model route selector only for status --- ods/ods-cli | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ods/ods-cli b/ods/ods-cli index 545d7b2dc5..e6b3b1d9bc 100755 --- a/ods/ods-cli +++ b/ods/ods-cli @@ -24,8 +24,6 @@ while [[ -L "$_source" ]]; do [[ "$_source" != /* ]] && _source="$_dir/$_source" done SCRIPT_DIR="$(cd "$(dirname "$_source")" && pwd)" -# shellcheck source=lib/preflight-llm-route.sh -. "$SCRIPT_DIR/lib/preflight-llm-route.sh" _resolve_cli_install_dir() { if [[ -n "${INSTALL_DIR:-}" ]]; then @@ -1327,6 +1325,10 @@ cmd_status() { cd "$INSTALL_DIR" load_env sr_load + # Only status needs the model-route selector. Keep unrelated CLI commands + # usable while repairing a partial install that is missing this library. + # shellcheck source=lib/preflight-llm-route.sh + . "$SCRIPT_DIR/lib/preflight-llm-route.sh" local flags_str flags_str=$(get_compose_flags) @@ -1449,6 +1451,8 @@ cmd_status_json() { cd "$INSTALL_DIR" sr_load load_env + # shellcheck source=lib/preflight-llm-route.sh + . "$SCRIPT_DIR/lib/preflight-llm-route.sh" local flags_str flags_str=$(get_compose_flags) From f579607d1faa8c44cb57cdb33294cb25336c3be6 Mon Sep 17 00:00:00 2001 From: Mike Bradley Date: Fri, 18 Sep 2026 00:21:55 -0400 Subject: [PATCH 26/52] fix(models): show unmatched runtime model without catalog claims --- .../services/dashboard-api/models.py | 12 +++--- .../dashboard-api/performance_oracle.py | 28 +++++++++++++ .../services/dashboard-api/routers/models.py | 5 +-- .../dashboard-api/tests/test_models.py | 39 +++++++++++++++++ .../tests/test_performance_oracle.py | 42 ++++++++++++++++++- .../services/dashboard/src/pages/Models.jsx | 15 +++++-- .../dashboard/src/pages/Models.test.jsx | 22 +++++++++- 7 files changed, 148 insertions(+), 15 deletions(-) diff --git a/ods/extensions/services/dashboard-api/models.py b/ods/extensions/services/dashboard-api/models.py index e6df38d922..0617685f6f 100644 --- a/ods/extensions/services/dashboard-api/models.py +++ b/ods/extensions/services/dashboard-api/models.py @@ -183,11 +183,11 @@ class ModelLibraryEntry(BaseModel): downloadUrl: Optional[str] = None downloadSha256: Optional[str] = None llmModelName: Optional[str] = None - size: str - sizeGb: float - vramRequired: float + size: Optional[str] + sizeGb: Optional[float] + vramRequired: Optional[float] estimatedRequired: Optional[float] = None - contextLength: int + contextLength: Optional[int] maxContextLength: Optional[int] = None contextOptions: list[dict[str, Any]] = Field(default_factory=list) specialty: str @@ -205,9 +205,9 @@ class ModelLibraryEntry(BaseModel): recommended: bool = False configured: bool = False recommendation: Optional[dict[str, Any]] = None - fitsVram: bool + fitsVram: Optional[bool] activationSupport: Optional[dict[str, Any]] = None - fitsCurrentVram: bool + fitsCurrentVram: Optional[bool] performance: Optional[dict[str, Any]] = None performanceLabel: Optional[str] = None diff --git a/ods/extensions/services/dashboard-api/performance_oracle.py b/ods/extensions/services/dashboard-api/performance_oracle.py index cb4e0d3ea2..e2daaea89e 100644 --- a/ods/extensions/services/dashboard-api/performance_oracle.py +++ b/ods/extensions/services/dashboard-api/performance_oracle.py @@ -11,6 +11,7 @@ from __future__ import annotations +import hashlib import json import os import platform @@ -1575,6 +1576,33 @@ def append_model(model: dict[str, Any], path: Path | None, status_if_not_loaded: } append_model(fallback, path, "downloaded") + if isinstance(loaded_model, str) and loaded_model.strip() and current_model_id is None: + # An external runtime can report a model that is neither in our catalog + # nor an inspectable local GGUF. Show what is actually running without + # borrowing a different quantization's size, fit, or activation claims. + response_models.append({ + "id": f"runtime-{hashlib.sha256(loaded_model.encode('utf-8')).hexdigest()[:12]}", + "name": loaded_model, + "gguf": None, + "downloadUrl": None, + "size": None, + "sizeGb": None, + "vramRequired": None, + "estimatedRequired": None, + "contextLength": context_length, + "specialty": "Runtime", + "description": "Reported as loaded by the model runtime; not an ODS catalog or inspected local model.", + "metadata": {"source": "runtime", "catalogSource": "runtime", "readable": False}, + "appCompatibility": {}, + "status": "loaded", + "recommended": False, + "configured": False, + "fitsVram": None, + "activationSupport": None, + "fitsCurrentVram": None, + "performance": None, + }) + return { "models": response_models, "gpu": gpu_data, diff --git a/ods/extensions/services/dashboard-api/routers/models.py b/ods/extensions/services/dashboard-api/routers/models.py index 2c83a168da..b1c4180a50 100644 --- a/ods/extensions/services/dashboard-api/routers/models.py +++ b/ods/extensions/services/dashboard-api/routers/models.py @@ -1358,8 +1358,8 @@ async def list_models(api_key: str = Depends(verify_api_key)): payload, _model_lifecycle_from_agent_status(agent_status), ) - if gpu_info and loaded_model and live_tps > 0: - loaded_entry = next((m for m in payload["models"] if m["status"] == "loaded"), None) or {} + loaded_entry = next((m for m in payload["models"] if m["status"] == "loaded"), None) or {} + if gpu_info and loaded_model and live_tps > 0 and loaded_entry.get("metadata", {}).get("source") != "runtime": signature = build_sample_signature( loaded_entry or {"id": loaded_model, "gguf": _read_active_model()}, gpu_info, @@ -1386,7 +1386,6 @@ async def list_models(api_key: str = Depends(verify_api_key)): payload["odsMode"] = ODS_MODE_EFFECTIVE payload["configuredMode"] = _configured_ods_mode() payload["llmBackend"] = LLM_BACKEND or "unknown" - loaded_entry = next((model for model in payload["models"] if model["status"] == "loaded"), None) payload["activationReadyModel"] = ( payload.get("currentModel") if loaded_entry diff --git a/ods/extensions/services/dashboard-api/tests/test_models.py b/ods/extensions/services/dashboard-api/tests/test_models.py index ad95ef234d..0bc462483a 100644 --- a/ods/extensions/services/dashboard-api/tests/test_models.py +++ b/ods/extensions/services/dashboard-api/tests/test_models.py @@ -1718,6 +1718,45 @@ def test_api_models_returns_full_catalog_without_fake_tokens(test_client, monkey assert payload["models"][0]["performance"]["source"] == "benchmark_required" +def test_api_models_reports_unmatched_external_runtime_without_fake_performance(test_client, monkeypatch, tmp_path): + models_router, install_dir, _data_dir = _patch_model_router_paths(monkeypatch, tmp_path) + _write_model_library(install_dir, [{ + "id": "qwen3.6-35b-a3b-ud-q4", + "name": "Qwen 3.6 35B-A3B", + "gguf_file": "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", + "size_mb": 21110, + "vram_required_gb": 24, + "context_length": 131072, + "quantization": "UD-Q4_K_M", + "specialty": "Quality", + "description": "Catalog quantization, not the observed external runtime.", + "llm_model_name": "qwen3.6-35b-a3b", + }]) + runtime_name = "Qwen3.6-35B-A3B-GGUF" + recorded = [] + monkeypatch.setattr(models_router, "get_gpu_info", lambda: _gpu()) + monkeypatch.setattr(models_router, "get_loaded_model", AsyncMock(return_value=runtime_name)) + monkeypatch.setattr(models_router, "get_llama_metrics", AsyncMock(return_value={"tokens_per_second": 42})) + monkeypatch.setattr(models_router, "get_llama_context_size", AsyncMock(return_value=None)) + monkeypatch.setattr(models_router, "record_model_performance", lambda *args, **kwargs: recorded.append((args, kwargs))) + + response = test_client.get("/api/models", headers=test_client.auth_headers) + + assert response.status_code == 200 + payload = response.json() + active = [entry for entry in payload["models"] if entry["status"] == "loaded"] + assert len(active) == 1 + assert active[0]["name"] == runtime_name + assert active[0]["metadata"]["source"] == "runtime" + assert active[0]["sizeGb"] is None + assert active[0]["vramRequired"] is None + assert active[0]["quantization"] is None + assert payload["currentModel"] is None + assert payload["activationReadyModel"] is None + assert payload["loadedModel"] == runtime_name + assert recorded == [] + + def test_download_model_rejects_while_bootstrap_upgrade_active(test_client, monkeypatch, tmp_path): models_router, install_dir, _data_dir = _patch_model_router_paths(monkeypatch, tmp_path) _write_model_library(install_dir, [ diff --git a/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py b/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py index 96ea40a739..622886d69d 100644 --- a/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py +++ b/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py @@ -2,7 +2,7 @@ from pathlib import Path from helpers import record_model_performance -from models import GPUInfo +from models import GPUInfo, ModelLibraryResponse from performance_oracle import ( build_models_payload, current_model_matches, @@ -162,6 +162,46 @@ def test_real_catalog_phi_models_have_exactly_one_loaded_identity(data_dir, tmp_ assert payload["currentModel"] == expected_id +def test_unmatched_runtime_model_is_visible_without_borrowing_catalog_metadata(data_dir, tmp_path): + install_dir = tmp_path / "ods" + install_dir.mkdir() + catalog_model = { + **_model(), + "id": "qwen3.6-35b-a3b-ud-q4", + "name": "Qwen 3.6 35B-A3B", + "gguf_file": "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", + "llm_model_name": "qwen3.6-35b-a3b", + "quantization": "UD-Q4_K_M", + } + runtime_name = "Qwen3.6-35B-A3B-GGUF" + + payload = build_models_payload( + _gpu(), runtime_name, 0, install_dir, data_dir, catalog=[catalog_model], evidence=[] + ) + loaded = [entry for entry in payload["models"] if entry["status"] == "loaded"] + + assert len(loaded) == 1 + assert loaded[0]["id"].startswith("runtime-") + assert loaded[0]["name"] == runtime_name + assert loaded[0]["metadata"]["source"] == "runtime" + assert loaded[0]["metadata"]["readable"] is False + for field in ("gguf", "downloadUrl", "size", "sizeGb", "vramRequired", + "estimatedRequired", "contextLength", "quantization", "architecture", + "fitsVram", "fitsCurrentVram", "activationSupport"): + assert loaded[0].get(field) is None, field + assert payload["currentModel"] is None + assert payload["loadedModel"] == runtime_name + assert payload["models"][0]["status"] != "loaded" + ModelLibraryResponse(**payload) + + matched = build_models_payload( + _gpu(), "qwen3.6-35b-a3b", 0, install_dir, data_dir, + catalog=[catalog_model], evidence=[], + ) + assert [entry["id"] for entry in matched["models"] if entry["status"] == "loaded"] == [catalog_model["id"]] + assert not any(entry["metadata"]["source"] == "runtime" for entry in matched["models"]) + + def test_benchmark_required_without_measurement_or_evidence(data_dir, tmp_path): install_dir = tmp_path / "ods" (install_dir / "data" / "models").mkdir(parents=True) diff --git a/ods/extensions/services/dashboard/src/pages/Models.jsx b/ods/extensions/services/dashboard/src/pages/Models.jsx index d9ca405c7b..f7fd58a5d5 100644 --- a/ods/extensions/services/dashboard/src/pages/Models.jsx +++ b/ods/extensions/services/dashboard/src/pages/Models.jsx @@ -812,13 +812,14 @@ function ModelTableRow({ }) { const isLoaded = model.status === 'loaded' || isCurrentModel const isDownloaded = model.status === 'downloaded' + const isRuntimeManaged = model.metadata?.source === 'runtime' const memory = getMemoryMeta(model, gpu) const compatibility = getCompatibilityMeta(model, memory, pixelMinimumContext) const speed = getSpeedDisplay(model) const tags = getModelTags(model, hermesMinimumContext) const iconTone = getIconTone(model, compatibility) const performanceBadge = getPerformanceBadge(model) - const runDisabledReason = getRunDisabledReason({ + const runDisabledReason = isRuntimeManaged ? null : getRunDisabledReason({ model, gpu, canActivateModels, @@ -835,7 +836,7 @@ function ModelTableRow({
{compatibility.label}{compatibility.detail}
- {isLoaded && } + {isLoaded && !isRuntimeManaged && }
Details

{model.description || 'No description available.'}

{tags.join(' · ')}

{performanceBadge &&

{performanceBadge.label}

}

{compatibility.label}: {compatibility.detail}

{runDisabledReason &&

{runDisabledReason}

}
@@ -876,7 +877,7 @@ function ModelTableRow({ onLoad={onLoad} onBenchmark={onBenchmark} /> - {isLoaded && ( + {isLoaded && !isRuntimeManaged && ( })} {loading &&

Loading models…

} {!loading && !installed.length && !showActiveFallback &&

No installed models found.

} {reason &&

{reason}

} {(localError || error) &&

{localError || error}

} -