From 8bfc068da9b5c7a1e1e761a72b63a430c5d51bdc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 19 Oct 2026 00:02:45 +0000 Subject: [PATCH] Pin every forced lineage to July v5; allowlist stale sidecar exemption The July v5 sha pin only fired when checkpoint_sha256 was empty, so a hand-written forced lineage with any fake checkpoint skipped it. Pin every forced: true lineage unless a matching promote.py manifest verifies the bundle. Marking any sidecar stale skipped run-id drift. Limit that exemption to mtnn_jacobian.json and mtnn_attr_pop.json. Co-authored-by: Cam Davis --- pipeline/served_model.py | 117 ++++++++++++++++++++++++------- scripts/check_served_model.py | 17 +++-- scripts/record_served_lineage.py | 3 +- tests/test_check_served_model.py | 39 +++++++++++ 4 files changed, 142 insertions(+), 34 deletions(-) diff --git a/pipeline/served_model.py b/pipeline/served_model.py index 2d20ca0a..31a460d6 100644 --- a/pipeline/served_model.py +++ b/pipeline/served_model.py @@ -28,13 +28,15 @@ sha of the served row order, and the metrics copied from the promoted manifest and every JSON sidecar exported from the same bundle (SIDECARS) records the -run_id it came from under "lineage", unless it is marked stale / -not_from_served_net (the 48-d jacobian and attr_pop still served from -2026-07-11/14). problems() is the whole check; scripts/check_served_model.py -runs it on assets/ and public/assets/. On master, scripts/record_served_lineage.py -wrote that lineage for the July v5 incumbent (run july-v5-a4918f09) from the -served bytes. A forced adoption without a promoted checkpoint is pinned to -INCUMBENT_F32_SHA256. +run_id it came from under "lineage", unless it is one of +STALE_SIDECAR_ALLOWLIST (the 48-d jacobian and attr_pop still served from +2026-07-11/14) and marked stale / not_from_served_net. A stale mark on any +other sidecar does not skip the run-id check. problems() is the whole check; +scripts/check_served_model.py runs it on assets/ and public/assets/. On master, +scripts/record_served_lineage.py wrote that lineage for the July v5 incumbent +(run july-v5-a4918f09) from the served bytes. Every forced: true lineage is +pinned to INCUMBENT_F32_SHA256 unless a matching promote.py manifest verifies +the bundle. Stdlib only, so the CI job that runs it needs no install. """ @@ -55,11 +57,17 @@ # git rev-parse HEAD:assets/mtnn_embeddings.f32 on origin/master (a4918f09). # Forced adoption (record_served_lineage.py) is pinned to these bytes so a -# hand-written forced lineage cannot bless a different f32. promote.py --force -# of a real bundle still carries checkpoint_sha256 and is not this path. +# hand-written forced lineage cannot bless a different f32. A matching +# promote.py manifest on disk is the only other way a forced lineage may +# name a different f32 (the real --force path). INCUMBENT_F32_SHA256 = "2574ef581ac3d3ee0f793447c26c5c48b82ec309985d6afcf177ed08510458f4" INCUMBENT_RUN_ID = "july-v5-a4918f09" +# Only these two files may skip the sidecar run-id drift check by marking +# stale / not_from_served_net. Any other sidecar that sets those flags still +# has to name the served run. +STALE_SIDECAR_ALLOWLIST = frozenset({"mtnn_jacobian.json", "mtnn_attr_pop.json"}) + # Metrics the promoted manifest carries (copied from the training report by # promote.py). The meta and the lineage file both copy them from there. METRIC_KEYS = ( @@ -160,26 +168,83 @@ def sidecar_not_from_served_net(lineage: Mapping[str, Any] | None) -> bool: return bool(lineage.get("stale") or lineage.get("not_from_served_net")) -def forced_adoption_pin_problem(lineage: Mapping[str, Any]) -> str | None: - """Forced adoption without a promoted checkpoint is only valid for the July v5 f32. +def sidecar_skips_run_id_check(name: str, lineage: Mapping[str, Any] | None) -> bool: + """Only the allowlisted 48-d jacobian and attr_pop may skip run-id drift.""" + return name in STALE_SIDECAR_ALLOWLIST and sidecar_not_from_served_net(lineage) + + +def _candidate_promoted_dirs(assets: Path) -> tuple[Path, ...]: + """Conventional places promote.py writes a bundle, relative to an assets dir.""" + parent = assets.parent + return ( + parent / "pipeline" / "data" / "promoted", + parent.parent / "pipeline" / "data" / "promoted", + parent / "data" / "promoted", + ) + + +def matching_promote_manifest(assets: Path, lineage: Mapping[str, Any]) -> bool: + """True when lineage names a real promote.py bundle whose manifest hashes match. + + Fail-closed: a missing run_id, missing manifest_sha256, missing files, or + any hash/run mismatch is not a match. A hand-written checkpoint_sha256 + string is not enough. + """ + run_id = lineage.get("run_id") + man_sha = lineage.get("manifest_sha256") + ckpt_sha = lineage.get("checkpoint_sha256") + if not isinstance(run_id, str) or not run_id: + return False + if not isinstance(man_sha, str) or not man_sha: + return False + if not isinstance(ckpt_sha, str) or not ckpt_sha: + return False + for promoted in _candidate_promoted_dirs(assets): + man_path = promoted / run_id / "manifest.json" + current_path = promoted / "CURRENT.json" + if not (man_path.is_file() and current_path.is_file()): + continue + try: + if sha256_file(man_path) != man_sha: + continue + man = json.loads(man_path.read_text(encoding="utf-8")) + current = json.loads(current_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if not isinstance(man, dict) or man.get("run_id") != run_id: + continue + if current.get("run_id") != run_id or current.get("manifest_sha256") != man_sha: + continue + files = man.get("files") if isinstance(man.get("files"), dict) else {} + rec = files.get("checkpoint") if isinstance(files.get("checkpoint"), dict) else {} + if rec.get("sha256") != ckpt_sha: + continue + return True + return False + + +def forced_adoption_pin_problem(lineage: Mapping[str, Any], assets: Path | None = None) -> str | None: + """Every forced: true lineage must name the July v5 f32, fail-closed. - promote.py --force of a real bundle writes checkpoint_sha256 and is not - pinned here (tests/test_e2e_smoke.py). A hand-written forced lineage, or - adoption.kind == served-incumbent, must name INCUMBENT_F32_SHA256. + checkpoint_sha256 being present used to skip this pin; a hand-written + forced lineage with any fake string then blessed a different f32. Now + forced: true is pinned regardless of checkpoint_sha256 or kind. A matching + promote.py manifest on disk (tests/test_e2e_smoke.py) is the only + alternative. adoption.kind == served-incumbent is also pinned, forced or not. """ adoption = lineage.get("adoption") if isinstance(lineage.get("adoption"), dict) else {} - is_adoption = adoption.get("kind") == "served-incumbent" or ( - bool(lineage.get("forced")) and not lineage.get("checkpoint_sha256") - ) - if not is_adoption: + forced = bool(lineage.get("forced")) + if not forced and adoption.get("kind") != "served-incumbent": return None sha = lineage.get("embedding_f32_sha256") - if sha != INCUMBENT_F32_SHA256: - return ( - f"{LINEAGE}: forced adoption is pinned to the July v5 incumbent " - f"sha256 {INCUMBENT_F32_SHA256[:12]}, not {str(sha)[:12]}" - ) - return None + if sha == INCUMBENT_F32_SHA256: + return None + if forced and assets is not None and matching_promote_manifest(assets, lineage): + return None + return ( + f"{LINEAGE}: forced adoption is pinned to the July v5 incumbent " + f"sha256 {INCUMBENT_F32_SHA256[:12]}, not {str(sha)[:12]}" + ) def _load(path: Path) -> tuple[Any, str | None]: @@ -272,7 +337,7 @@ def problems(assets: Path) -> list[str]: return out if lineage.get("schema") != LINEAGE_SCHEMA: out.append(f"{LINEAGE}: schema {lineage.get('schema')!r}, expected {LINEAGE_SCHEMA}") - pin = forced_adoption_pin_problem(lineage) + pin = forced_adoption_pin_problem(lineage, assets) if pin: out.append(pin) run_id = lineage.get("run_id") @@ -310,7 +375,7 @@ def problems(assets: Path) -> list[str]: continue side_lin = doc.get("lineage") if isinstance(doc, dict) else None side = side_lin.get("run_id") if isinstance(side_lin, dict) else None - if sidecar_not_from_served_net(side_lin if isinstance(side_lin, dict) else None): + if sidecar_skips_run_id_check(name, side_lin if isinstance(side_lin, dict) else None): if side == run_id: out.append(f"{name} is marked not-from-served-net but claims run {run_id!r}") continue diff --git a/scripts/check_served_model.py b/scripts/check_served_model.py index 09a50e85..baecc9ef 100644 --- a/scripts/check_served_model.py +++ b/scripts/check_served_model.py @@ -25,13 +25,16 @@ On master the served bundle is the July v5 export (f32 git blob a4918f09). scripts/record_served_lineage.py recorded mtnn_lineage.json from those bytes -(run july-v5-a4918f09) so this check verifies the deployed tree. A forced -adoption (forced: true and no checkpoint_sha256, or adoption.kind -served-incumbent) is pinned to that f32's sha256: a hand-written forced -lineage for a different model fails. Sidecars marked stale / -not_from_served_net must not claim the served run id (the 48-d jacobian -and attr_pop). It still fails a swapped f32, extra meta keys, a keys-hash -mismatch, live sidecar run-id drift, or assets/ vs public/assets/ skew. +(run july-v5-a4918f09) so this check verifies the deployed tree. Every +forced: true lineage is pinned to that f32's sha256, regardless of +checkpoint_sha256 or kind: a hand-written forced lineage for a different +model fails. A matching promote.py manifest on disk is the only alternative +(the real --force path). Only mtnn_jacobian.json and mtnn_attr_pop.json may +skip the sidecar run-id check by marking stale / not_from_served_net; any +other sidecar marked stale still fails on run-id drift. Those two must not +claim the served run id. It still fails a swapped f32, extra meta keys, a +keys-hash mismatch, live sidecar run-id drift, or assets/ vs public/assets/ +skew. A replacement model is not recorded that way: re-promote (pipeline/promote.py), re-export through rebuild_all.py's export stage, re-stamp with scripts/stamp_assets.py, then mirror with scripts/sync_public.py. diff --git a/scripts/record_served_lineage.py b/scripts/record_served_lineage.py index d23c7dc4..a7f4ca8f 100755 --- a/scripts/record_served_lineage.py +++ b/scripts/record_served_lineage.py @@ -14,7 +14,8 @@ It refuses any f32 other than the pinned incumbent. A replacement model still has to go through pipeline/promote.py + export_mtnn_embeddings.py. -check_served_model.py also enforces that pin on forced adoption. +check_served_model.py also enforces that pin on every forced: true lineage, +regardless of checkpoint_sha256. Stdlib only. diff --git a/tests/test_check_served_model.py b/tests/test_check_served_model.py index 7161059e..1a463961 100644 --- a/tests/test_check_served_model.py +++ b/tests/test_check_served_model.py @@ -66,6 +66,7 @@ def test_the_committed_served_bundle_passes(): def test_the_committed_jacobian_and_attr_pop_are_marked_not_from_served_net(): + assert sm.STALE_SIDECAR_ALLOWLIST == frozenset({"mtnn_jacobian.json", "mtnn_attr_pop.json"}) for name in ("mtnn_jacobian.json", "mtnn_attr_pop.json"): lin = json.loads((ROOT / "assets" / name).read_text(encoding="utf-8"))["lineage"] assert lin["stale"] is True and lin["not_from_served_net"] is True @@ -88,6 +89,21 @@ def test_a_forced_lineage_with_a_different_sha_fails(site): assert csm.main(["--root", str(site)]) == 1 +def test_a_forced_lineage_with_a_fake_checkpoint_sha_fails(site): + """forced: true pins the July v5 sha even when checkpoint_sha256 is a non-empty fake.""" + for rel in csm.SERVED_DIRS: + path = site / rel / sm.LINEAGE + lin = json.loads(path.read_text(encoding="utf-8")) + lin["forced"] = True + lin["force_reason"] = "sneak in a new model" + lin["checkpoint_sha256"] = "a" * 64 + path.write_text(json.dumps(lin), encoding="utf-8") + why = problems(site) + assert "forced adoption is pinned to the July v5 incumbent" in why + assert str(lin["embedding_f32_sha256"])[:12] in why + assert csm.main(["--root", str(site)]) == 1 + + def test_a_stale_sidecar_claiming_the_served_run_fails(site): (site / "assets" / "mtnn_jacobian.json").write_text( json.dumps({"lineage": {"run_id": "r1", "stale": True, "not_from_served_net": True}}), @@ -96,6 +112,29 @@ def test_a_stale_sidecar_claiming_the_served_run_fails(site): assert "assets/mtnn_jacobian.json is marked not-from-served-net but claims run 'r1'" in problems(site) +def test_a_non_allowlisted_stale_sidecar_with_drifted_run_id_fails(site): + """Marking a sidecar stale is not a self-exemption; only the 48-d allowlist skips drift.""" + (site / "assets" / "mtnn_map.json").write_text( + json.dumps({"lineage": {"run_id": "other-run", "stale": True, "not_from_served_net": True}}), + encoding="utf-8", + ) + assert "assets/mtnn_map.json is from run 'other-run', the served embedding from 'r1'" in problems(site) + + +def test_allowlisted_stale_sidecars_with_drifted_run_id_pass(site): + """The 48-d jacobian and attr_pop may still skip run-id drift when marked stale.""" + for name, rid in ( + ("mtnn_jacobian.json", "pre-v5-48d-2026-07-14"), + ("mtnn_attr_pop.json", "pre-v5-48d-2026-07-11"), + ): + for rel in csm.SERVED_DIRS: + (site / rel / name).write_text( + json.dumps({"lineage": {"run_id": rid, "stale": True, "not_from_served_net": True}}), + encoding="utf-8", + ) + assert csm.check(site) == [] + + def test_record_served_lineage_refuses_a_different_f32(tmp_path, monkeypatch, capsys): spec = importlib.util.spec_from_file_location( "record_served_lineage", ROOT / "scripts" / "record_served_lineage.py"