Close forced-lineage and stale-sidecar gaps in the served-model check

pipeline/served_model.py

@@ -28,13 +28,15 @@
                         sha of the served row order, and the metrics copied
                         from the promoted manifest
 and every JSON sidecar exported from the same bundle (SIDECARS) records the
-run_id it came from under "lineage", unless it is marked stale /
-not_from_served_net (the 48-d jacobian and attr_pop still served from
-2026-07-11/14). problems() is the whole check; scripts/check_served_model.py
-runs it on assets/ and public/assets/. On master, scripts/record_served_lineage.py
-wrote that lineage for the July v5 incumbent (run july-v5-a4918f09) from the
-served bytes. A forced adoption without a promoted checkpoint is pinned to
-INCUMBENT_F32_SHA256.
+run_id it came from under "lineage", unless it is one of
+STALE_SIDECAR_ALLOWLIST (the 48-d jacobian and attr_pop still served from
+2026-07-11/14) and marked stale / not_from_served_net. A stale mark on any
+other sidecar does not skip the run-id check. problems() is the whole check;
+scripts/check_served_model.py runs it on assets/ and public/assets/. On master,
+scripts/record_served_lineage.py wrote that lineage for the July v5 incumbent
+(run july-v5-a4918f09) from the served bytes. Every forced: true lineage is
+pinned to INCUMBENT_F32_SHA256 unless a matching promote.py manifest verifies
+the bundle.
 
 Stdlib only, so the CI job that runs it needs no install.
 """
@@ -55,11 +57,17 @@
 
 # git rev-parse HEAD:assets/mtnn_embeddings.f32 on origin/master (a4918f09).
 # Forced adoption (record_served_lineage.py) is pinned to these bytes so a
-# hand-written forced lineage cannot bless a different f32. promote.py --force
-# of a real bundle still carries checkpoint_sha256 and is not this path.
+# hand-written forced lineage cannot bless a different f32. A matching
+# promote.py manifest on disk is the only other way a forced lineage may
+# name a different f32 (the real --force path).
 INCUMBENT_F32_SHA256 = "2574ef581ac3d3ee0f793447c26c5c48b82ec309985d6afcf177ed08510458f4"
 INCUMBENT_RUN_ID = "july-v5-a4918f09"
 
+# Only these two files may skip the sidecar run-id drift check by marking
+# stale / not_from_served_net. Any other sidecar that sets those flags still
+# has to name the served run.
+STALE_SIDECAR_ALLOWLIST = frozenset({"mtnn_jacobian.json", "mtnn_attr_pop.json"})
+
 # Metrics the promoted manifest carries (copied from the training report by
 # promote.py). The meta and the lineage file both copy them from there.
 METRIC_KEYS = (
@@ -160,26 +168,83 @@ def sidecar_not_from_served_net(lineage: Mapping[str, Any] | None) -> bool:
     return bool(lineage.get("stale") or lineage.get("not_from_served_net"))
 
 
-def forced_adoption_pin_problem(lineage: Mapping[str, Any]) -> str | None:
-    """Forced adoption without a promoted checkpoint is only valid for the July v5 f32.
+def sidecar_skips_run_id_check(name: str, lineage: Mapping[str, Any] | None) -> bool:
+    """Only the allowlisted 48-d jacobian and attr_pop may skip run-id drift."""
+    return name in STALE_SIDECAR_ALLOWLIST and sidecar_not_from_served_net(lineage)
+
+
+def _candidate_promoted_dirs(assets: Path) -> tuple[Path, ...]:
+    """Conventional places promote.py writes a bundle, relative to an assets dir."""
+    parent = assets.parent
+    return (
+        parent / "pipeline" / "data" / "promoted",
+        parent.parent / "pipeline" / "data" / "promoted",
+        parent / "data" / "promoted",
+    )
+
+
+def matching_promote_manifest(assets: Path, lineage: Mapping[str, Any]) -> bool:
+    """True when lineage names a real promote.py bundle whose manifest hashes match.
+
+    Fail-closed: a missing run_id, missing manifest_sha256, missing files, or
+    any hash/run mismatch is not a match. A hand-written checkpoint_sha256
+    string is not enough.
+    """
+    run_id = lineage.get("run_id")
+    man_sha = lineage.get("manifest_sha256")
+    ckpt_sha = lineage.get("checkpoint_sha256")
+    if not isinstance(run_id, str) or not run_id:
+        return False
+    if not isinstance(man_sha, str) or not man_sha:
+        return False
+    if not isinstance(ckpt_sha, str) or not ckpt_sha:
+        return False
+    for promoted in _candidate_promoted_dirs(assets):
+        man_path = promoted / run_id / "manifest.json"
+        current_path = promoted / "CURRENT.json"
+        if not (man_path.is_file() and current_path.is_file()):
+            continue
+        try:
+            if sha256_file(man_path) != man_sha:
+                continue
+            man = json.loads(man_path.read_text(encoding="utf-8"))
+            current = json.loads(current_path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            continue
+        if not isinstance(man, dict) or man.get("run_id") != run_id:
+            continue
+        if current.get("run_id") != run_id or current.get("manifest_sha256") != man_sha:
+            continue
+        files = man.get("files") if isinstance(man.get("files"), dict) else {}
+        rec = files.get("checkpoint") if isinstance(files.get("checkpoint"), dict) else {}
+        if rec.get("sha256") != ckpt_sha:
+            continue
+        return True
+    return False
+
+
+def forced_adoption_pin_problem(lineage: Mapping[str, Any], assets: Path | None = None) -> str | None:
+    """Every forced: true lineage must name the July v5 f32, fail-closed.
 
-    promote.py --force of a real bundle writes checkpoint_sha256 and is not
-    pinned here (tests/test_e2e_smoke.py). A hand-written forced lineage, or
-    adoption.kind == served-incumbent, must name INCUMBENT_F32_SHA256.
+    checkpoint_sha256 being present used to skip this pin; a hand-written
+    forced lineage with any fake string then blessed a different f32. Now
+    forced: true is pinned regardless of checkpoint_sha256 or kind. A matching
+    promote.py manifest on disk (tests/test_e2e_smoke.py) is the only
+    alternative. adoption.kind == served-incumbent is also pinned, forced or not.
     """
     adoption = lineage.get("adoption") if isinstance(lineage.get("adoption"), dict) else {}
-    is_adoption = adoption.get("kind") == "served-incumbent" or (
-        bool(lineage.get("forced")) and not lineage.get("checkpoint_sha256")
-    )
-    if not is_adoption:
+    forced = bool(lineage.get("forced"))
+    if not forced and adoption.get("kind") != "served-incumbent":
         return None
     sha = lineage.get("embedding_f32_sha256")
-    if sha != INCUMBENT_F32_SHA256:
-        return (
-            f"{LINEAGE}: forced adoption is pinned to the July v5 incumbent "
-            f"sha256 {INCUMBENT_F32_SHA256[:12]}, not {str(sha)[:12]}"
-        )
-    return None
+    if sha == INCUMBENT_F32_SHA256:
+        return None
+    if forced and assets is not None and matching_promote_manifest(assets, lineage):
+        return None
+    return (
+        f"{LINEAGE}: forced adoption is pinned to the July v5 incumbent "
+        f"sha256 {INCUMBENT_F32_SHA256[:12]}, not {str(sha)[:12]}"
+    )
 
 
 def _load(path: Path) -> tuple[Any, str | None]:
@@ -272,7 +337,7 @@ def problems(assets: Path) -> list[str]:
         return out
     if lineage.get("schema") != LINEAGE_SCHEMA:
         out.append(f"{LINEAGE}: schema {lineage.get('schema')!r}, expected {LINEAGE_SCHEMA}")
-    pin = forced_adoption_pin_problem(lineage)
+    pin = forced_adoption_pin_problem(lineage, assets)
     if pin:
         out.append(pin)
     run_id = lineage.get("run_id")
@@ -310,7 +375,7 @@ def problems(assets: Path) -> list[str]:
             continue
         side_lin = doc.get("lineage") if isinstance(doc, dict) else None
         side = side_lin.get("run_id") if isinstance(side_lin, dict) else None
-        if sidecar_not_from_served_net(side_lin if isinstance(side_lin, dict) else None):
+        if sidecar_skips_run_id_check(name, side_lin if isinstance(side_lin, dict) else None):
             if side == run_id:
                 out.append(f"{name} is marked not-from-served-net but claims run {run_id!r}")
             continue

scripts/check_served_model.py

@@ -25,13 +25,16 @@
 
 On master the served bundle is the July v5 export (f32 git blob a4918f09).
 scripts/record_served_lineage.py recorded mtnn_lineage.json from those bytes
-(run july-v5-a4918f09) so this check verifies the deployed tree. A forced
-adoption (forced: true and no checkpoint_sha256, or adoption.kind
-served-incumbent) is pinned to that f32's sha256: a hand-written forced
-lineage for a different model fails. Sidecars marked stale /
-not_from_served_net must not claim the served run id (the 48-d jacobian
-and attr_pop). It still fails a swapped f32, extra meta keys, a keys-hash
-mismatch, live sidecar run-id drift, or assets/ vs public/assets/ skew.
+(run july-v5-a4918f09) so this check verifies the deployed tree. Every
+forced: true lineage is pinned to that f32's sha256, regardless of
+checkpoint_sha256 or kind: a hand-written forced lineage for a different
+model fails. A matching promote.py manifest on disk is the only alternative
+(the real --force path). Only mtnn_jacobian.json and mtnn_attr_pop.json may
+skip the sidecar run-id check by marking stale / not_from_served_net; any
+other sidecar marked stale still fails on run-id drift. Those two must not
+claim the served run id. It still fails a swapped f32, extra meta keys, a
+keys-hash mismatch, live sidecar run-id drift, or assets/ vs public/assets/
+skew.
 A replacement model is not recorded that way: re-promote (pipeline/promote.py),
 re-export through rebuild_all.py's export stage, re-stamp with
 scripts/stamp_assets.py, then mirror with scripts/sync_public.py.

scripts/record_served_lineage.py

@@ -14,7 +14,8 @@
 
 It refuses any f32 other than the pinned incumbent. A replacement model still
 has to go through pipeline/promote.py + export_mtnn_embeddings.py.
-check_served_model.py also enforces that pin on forced adoption.
+check_served_model.py also enforces that pin on every forced: true lineage,
+regardless of checkpoint_sha256.
 
 Stdlib only.
 

tests/test_check_served_model.py

@@ -66,6 +66,7 @@ def test_the_committed_served_bundle_passes():
 
 
 def test_the_committed_jacobian_and_attr_pop_are_marked_not_from_served_net():
+    assert sm.STALE_SIDECAR_ALLOWLIST == frozenset({"mtnn_jacobian.json", "mtnn_attr_pop.json"})
     for name in ("mtnn_jacobian.json", "mtnn_attr_pop.json"):
         lin = json.loads((ROOT / "assets" / name).read_text(encoding="utf-8"))["lineage"]
         assert lin["stale"] is True and lin["not_from_served_net"] is True
@@ -88,6 +89,21 @@ def test_a_forced_lineage_with_a_different_sha_fails(site):
     assert csm.main(["--root", str(site)]) == 1
 
 
+def test_a_forced_lineage_with_a_fake_checkpoint_sha_fails(site):
+    """forced: true pins the July v5 sha even when checkpoint_sha256 is a non-empty fake."""
+    for rel in csm.SERVED_DIRS:
+        path = site / rel / sm.LINEAGE
+        lin = json.loads(path.read_text(encoding="utf-8"))
+        lin["forced"] = True
+        lin["force_reason"] = "sneak in a new model"
+        lin["checkpoint_sha256"] = "a" * 64
+        path.write_text(json.dumps(lin), encoding="utf-8")
+    why = problems(site)
+    assert "forced adoption is pinned to the July v5 incumbent" in why
+    assert str(lin["embedding_f32_sha256"])[:12] in why
+    assert csm.main(["--root", str(site)]) == 1
+
+
 def test_a_stale_sidecar_claiming_the_served_run_fails(site):
     (site / "assets" / "mtnn_jacobian.json").write_text(
         json.dumps({"lineage": {"run_id": "r1", "stale": True, "not_from_served_net": True}}),
@@ -96,6 +112,29 @@ def test_a_stale_sidecar_claiming_the_served_run_fails(site):
     assert "assets/mtnn_jacobian.json is marked not-from-served-net but claims run 'r1'" in problems(site)
 
 
+def test_a_non_allowlisted_stale_sidecar_with_drifted_run_id_fails(site):
+    """Marking a sidecar stale is not a self-exemption; only the 48-d allowlist skips drift."""
+    (site / "assets" / "mtnn_map.json").write_text(
+        json.dumps({"lineage": {"run_id": "other-run", "stale": True, "not_from_served_net": True}}),
+        encoding="utf-8",
+    )
+    assert "assets/mtnn_map.json is from run 'other-run', the served embedding from 'r1'" in problems(site)
+
+
+def test_allowlisted_stale_sidecars_with_drifted_run_id_pass(site):
+    """The 48-d jacobian and attr_pop may still skip run-id drift when marked stale."""
+    for name, rid in (
+        ("mtnn_jacobian.json", "pre-v5-48d-2026-07-14"),
+        ("mtnn_attr_pop.json", "pre-v5-48d-2026-07-11"),
+    ):
+        for rel in csm.SERVED_DIRS:
+            (site / rel / name).write_text(
+                json.dumps({"lineage": {"run_id": rid, "stale": True, "not_from_served_net": True}}),
+                encoding="utf-8",
+            )
+    assert csm.check(site) == []
+
+
 def test_record_served_lineage_refuses_a_different_f32(tmp_path, monkeypatch, capsys):
     spec = importlib.util.spec_from_file_location(
         "record_served_lineage", ROOT / "scripts" / "record_served_lineage.py"