diff --git a/.github/workflows/run_pytest.yml b/.github/workflows/run_pytest.yml index e8dd1b2..af236e5 100644 --- a/.github/workflows/run_pytest.yml +++ b/.github/workflows/run_pytest.yml @@ -78,14 +78,37 @@ jobs: # into silence. Without this step a partial install silently shrinks the suite # and CI still reports green. Guarded by # tests/test_falsification_extras_actually_installed.py. + # Also asserts the resolver did NOT land in the floor's major: the floor step below + # covers that end, so this leg must cover the other. Read from pyproject through the + # one reader, not a literal. A deliberate false red: if every release above the + # floor's major were yanked, this fails loudly rather than silently testing 1.x twice. - name: Verify optional extras are installed - run: poetry run python -c "import views_frames" + run: poetry run python -c "import views_frames, importlib.metadata as m; from tests.test_falsification_extras_actually_installed import _views_frames_specifier; from packaging.specifiers import SpecifierSet; v = m.version('views-frames'); spec = SpecifierSet(_views_frames_specifier()); floor = min(str(s.version) for s in spec if s.operator == '>='); print('resolved views-frames', v, 'declared', spec, 'floor', floor); assert spec.contains(v), 'resolved outside the declared range'; assert v.split('.')[0] != floor.split('.')[0], 'the resolver picked the floor major; the other end of the range would go untested'" - name: Run tests run: | set -e poetry run pytest tests/ + # The `frames` extra admits two views-frames majors (>=1.10.2,<3; #91). The step + # above runs on whatever the resolver picks (asserted not to be the floor's major + # in the verify step), so the FLOOR would be untested. This reads the floor through + # the ONE reader the guards use (no second parser, no literal to drift), pins it + # with `--no-deps` so pip cannot move other pins, proves the environment consistent + # and the module importable, and runs the whole suite on it. Nothing after this + # step touches Python, so nothing is restored; + # a later Python-touching step must reinstall the extras first (the held workflow + # text forces that review). Guarded by the held workflow text. + - name: Run the suite on the views-frames floor + run: | + set -e + FLOOR=$(poetry run python -c "from tests.test_falsification_extras_actually_installed import _views_frames_specifier; from packaging.specifiers import SpecifierSet; print(min(str(s.version) for s in SpecifierSet(_views_frames_specifier()) if s.operator == '>='))") + echo "views-frames floor from pyproject.toml: $FLOOR" + poetry run pip install --quiet "views-frames==$FLOOR" --no-deps + poetry run pip check + poetry run python -c "import views_frames, importlib.metadata as m, sys; from views_frames import FrameMetadata; assert m.version('views-frames') == sys.argv[1], m.version('views-frames')" "$FLOOR" + poetry run pytest tests/ -q + # Structural documentation checks (CIC references, cross-ADR integrity, unfilled # placeholders). Complements tests/test_documentation_contracts.py, which asserts # that documented claims match code. Neither ran in CI before 2026-08-02, which is diff --git a/CHANGELOG.md b/CHANGELOG.md index f362b77..80a9e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,68 @@ _Nothing yet._ --- +## [2.1.0] — 2026-09-19 + +One constraint changes: the `frames` extra no longer caps views-frames below 2.0.0. The +rest of the release is the guards and CI legs that make that measured, and a sixth item +on the ADR-022 §7 release checklist so a stale cap is noticed at a release rather than by +a consumer. Nothing this package emits changes. Issue #91, reported by views-pipeline-core's +3.3.0 range review. + +### Changed + +- **The `frames` extra no longer excludes views-frames 2.x** (`>=1.10.2,<3`, was `<2`). + Reported by views-pipeline-core's 3.3.0 range review (#91): views-frames 2.0.0 has been + published since 2026-08-18 and this cap, carried into any environment that requests the + extra, excluded it. What this changes is this package's own contract only — a consumer + whose own bounds exclude views-frames 2.x must widen those too before it resolves. + Measured before widening: the full suite (at 2.0.0's count) on views-frames 1.10.2, + 1.11.0 and 2.0.0, and a `MetricFrame` saved under either major loads under the other + with identical rows and values. Safe because this package's only runtime import from + views-frames is the `FrameMetadata` dataclass (its tests also call the envelope contract + `views_frames.conformance.assert_frame_envelope`, whose body is unchanged between the + majors); views-frames 2.0.0's ADR-028 changes (index type, read-only `.values`, + `map_estimate` refusals) never reach it. Guards now pin the declared range and that + import surface, and CI runs the whole suite on the range's floor as well as on the + resolved version, asserting the resolved one is the 2.x end. Nothing changes in what + this package emits; MINOR under ADR-022 §5. + +### Release checklist (ADR-022 §7) + +- [x] **Does this release do anything rule 2 governs — remove an `__all__` symbol, remove + a supported config key, narrow an accepted input, or change a raised exception type? + If so, did a `DeprecationWarning` ship at least one release ago?** No. One dependency + constraint widens; no symbol, key, input or exception type changes. +- [x] **Does this release make previously-accepted input fail? If so, are the release + notes explicit, and have known consumers been notified?** No input changes. The one + consumer that requests the `frames` extra, views-reporting, was notified **before the + tag** on 2026-09-19 (views-platform/views-reporting#289, comment 5738670537); the + comment records what their own views-frames bound was when observed. views-pipeline-core + filed #91. +- [x] **Does the version bump match the change class (rule 5)?** `2.0.0` → `2.1.0`, + MINOR. §1 does not count dependency ranges as public API; widening one removes an + exclusion and breaks nothing that resolved before. +- [x] **Do the release notes list every breaking change with its migration?** There is + no breaking change. +- [x] **Does `MetricFrame`'s format or axis vocabulary change? If so, has it been agreed + with views-reporting and views-pipeline-core?** No. Measured: a frame saved under + views-frames 1.10.2 loads under 2.0.0 and the reverse, identically; CI runs the whole + suite on both ends of the range. +- [x] **Has any dependency this package caps published a release outside the declared + range since the last release? If so, is the cap measured and deliberate, or stale?** + This is the release that answers "stale" for views-frames and fixes it. Working the + item honestly for the rest: **numpy `<2.0.0` is stale.** Measured 2026-09-19 with + `pip install --dry-run --only-binary=:all:`: on Python 3.13 and 3.14, both inside our + declared `>=3.11,<3.15`, no numpy wheel satisfies `<2.0.0`, so this package cannot be + installed from wheels there (3.11 and 3.12 resolve). scipy 1.18 (2026-06-19) requires + numpy ≥ 2, so under our cap the resolver silently freezes scipy at 1.17.x. Not fixed in + this release — lifting the cap means measuring numpy 2.x — and tracked as #93 with + register C-41 re-tiered; until then, Python 3.13/3.14 users need a numpy 1.26 built from + source. `python <3.15`: 3.15.0 final is due 2026-10-01; #93 covers it. Dev group: + `pandas <3` has releases above it; dev-only, no consumer effect. + +--- + ## [2.0.0] — 2026-09-18 **MAJOR** (ADR-022 §5): this release removes the surface 1.1.0 deprecated, one release diff --git a/documentation/ADRs/022_evolution_and_stability.md b/documentation/ADRs/022_evolution_and_stability.md index 641be5b..c4611b8 100644 --- a/documentation/ADRs/022_evolution_and_stability.md +++ b/documentation/ADRs/022_evolution_and_stability.md @@ -127,6 +127,7 @@ This policy is checked at two points: - [ ] Does the version bump match the change class (rule 5)? - [ ] Do the release notes list every breaking change with its migration? - [ ] Does `MetricFrame`'s format or axis vocabulary change? If so, has it been agreed with views-reporting and views-pipeline-core? +- [ ] For every capped entry in `pyproject.toml` — `python` and the dev group included, not a fixed list — has the dependency published a release outside the declared range since the last release, or a release that requires something another of our caps excludes? If so, is the cap measured and deliberate, or stale? *(Added 2026-09-19, #91: the `frames` extra's `<2` cap outlived views-frames 2.0.0 by a month and was noticed by a consumer, not by a release of ours.)* ## Consequences diff --git a/documentation/CICs/MetricFrame.md b/documentation/CICs/MetricFrame.md index d6812bc..a9b82ed 100644 --- a/documentation/CICs/MetricFrame.md +++ b/documentation/CICs/MetricFrame.md @@ -2,7 +2,7 @@ **Status:** Active **Owner:** Evaluation Core -**Last reviewed:** 2026-08-02 +**Last reviewed:** 2026-09-19 **Related ADRs:** views-frames ADR-020 (contract home), ADR-041 (Output Schema), ADR-013 (Observability), ADR-015 (Degenerate/Empty Results), ADR-011 (Topology) --- @@ -42,7 +42,7 @@ It is a string-keyed value object — **not** a spatiotemporal `(time, unit)` fr - **`identifiers`**: dict containing exactly the keys in `AXES`, each a 1-D length-`N` array of strings. - **`metadata`**: optional `MetricFrameMetadata`; defaults to an empty one. - **Assumes** the caller has already decided what belongs in the frame. Construction is a structural gate, not a semantic one. -- **Requires** the optional `views-frames` dependency (`pip install views-evaluation[frames]`). The module is import-gated in `views_evaluation/__init__.py` on `find_spec`, so the core API stays importable without it (ADR-011 minimal core). +- **Requires** the optional `views-frames` dependency (`pip install views-evaluation[frames]`), any version in the range `pyproject.toml` declares (guarded by `TestViewsFramesRange`; measured on both ends, see CHANGELOG `[2.1.0]`). The module is import-gated in `views_evaluation/__init__.py` on `find_spec`, so the core API stays importable without it (ADR-011 minimal core). This package's only runtime import from views-frames is the `FrameMetadata` dataclass (six keyword fields, `to_dict`, `from_dict`); its second dependency is the envelope contract `views_frames.conformance.assert_frame_envelope`, called by the tests. `TestViewsFramesRange` pins the range and the one-name import surface, `tests/test_metric_frame.py::TestViewsFramesSurface` pins `FrameMetadata`'s behaviour on the installed major, and CI runs the whole suite on the range's floor as well as on the resolved version. --- diff --git a/pyproject.toml b/pyproject.toml index 6b497f4..05d1b4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "views_evaluation" -version = "2.0.0" +version = "2.1.0" description = "" authors = [ "Xiaolong Sun ", @@ -13,13 +13,14 @@ readme = "README.md" python = ">=3.11,<3.15" numpy = "^1.26.4" scipy = "^1.11" # Level-0 kernels (EMD, Pearson); ADR-011 permits numpy and scipy -views-frames = {version = ">=1.10.2,<2", optional = true} +views-frames = {version = ">=1.10.2,<3", optional = true} # both majors measured 2026-09-19 (#91); the only name used is FrameMetadata [tool.poetry.extras] frames = ["views-frames"] [tool.poetry.group.dev.dependencies] pytest = "^9.0.3" +packaging = ">=22" # Dev-only: the dependency-range guards and CI floor step parse specifiers with it; declared, not borrowed from pytest ruff = "*" properscoring = "^0.1.0" # Dev-only: used as test oracle for CRPS parity verification scikit-learn = "^1.6.0" # Dev-only: test oracle for AP/MTD parity verification (left runtime 2026-09-18, C-05) diff --git a/reports/technical_risk_register.md b/reports/technical_risk_register.md index 7cb8795..657e9e2 100644 --- a/reports/technical_risk_register.md +++ b/reports/technical_risk_register.md @@ -1,6 +1,6 @@ # Technical Risk Register — views-evaluation -**Last updated:** 2026-09-18 +**Last updated:** 2026-09-19 **Total open concerns:** 14 **Governing ADR:** ADR-023 **Citation convention:** `Location` fields name files and symbols (functions, classes, sections), not line numbers — line numbers drift as soon as anything is inserted above them. @@ -14,6 +14,32 @@ > review then found four gaps in the rewrite (linked-worktree `commondir`, locale-dependent > reads, unchecked ref contents, an unpinned name gate) — all fixed and tested before merge. 18 → 17. +> **2026-09-19 — release 2.1.0.** Carries #91 only. First release to answer ADR-022 §7's new sixth +> checklist item (stale caps): views-frames fixed, numpy and scipy caps noted as measured (C-41). +> §3.2 notification to views-reporting before the tag (#289, comment 5738670537), stating that their +> own direct views-frames bound still excludes 2.x. The release commit's guard audit (63 mutations) +> found the specific text guards weak behind the held-text backstop: nine exit-status evasions the +> suppressed list did not name (all proven to exit 0 under `bash -e`), a specifier reader that took the +> first of several declarations, and floor-step checks satisfied by tokens in the wrong place. All +> closed — the list carries the nine forms, the reader requires exactly one canonical declaration, the +> floor guard is structural (derivation → install → check → proof → pytest, FLOOR assigned once) — and +> re-verified red. Count unchanged. + +> **2026-09-19 — #91: the `frames` extra widened to views-frames `<3`.** views-pipeline-core's 3.3.0 +> range review found that our `<2` cap excluded views-frames 2.0.0 (published 2026-08-18) from any +> environment requesting the extra. (The review of this change found that the report's premise — that +> our cap was the *only* thing holding reporting environments on 1.x — was false: that consumer also +> declares its own `<2.0.0`; so the note below states only this repo's contract.) Measured on +> 1.10.2, 1.11.0 and 2.0.0 (full suite green on each; cross-major save/load identical) and widened; +> guards pin the range, the one-name import surface and `FrameMetadata`'s behaviour; CI runs the whole +> suite on the floor and asserts the resolved version is the 2.x end. Recorded under C-38 as its mirror +> case, C-38's trigger widened, C-41 updated, and ADR-022 §7 gained a checklist line so a capped +> dependency's new major is noticed at the next release rather than by a consumer. The guard audit +> found survivors on four of the new guards (a PEP 621 table read second, `seed=0` dropped by a falsy +> `to_dict`, an environment marker on the entry, `getattr`/re-aliasing on a views_frames alias, and +> floor-step checks that a `-k` or `|| true` slipped past), all closed and re-verified red. The release review then found the floor-step guard's `continue-on-error` check vacuous (it read a key the parser never emits) and its derivation check token-matchable; both replaced — the step now calls the one specifier reader and the guard asserts the call, the parser's own flag, the full suppressed-exit list, `--no-deps`, `pip check` and an import proof on the floor. Ships as +> 2.1.0 (MINOR). Count unchanged. + > **2026-09-18 — release 2.0.0 (S8, #72).** The MAJOR that ends the pandas-free epic: `to_dataframe()` > and the `dataframe` extra gone one release cycle after 1.1.0's `DeprecationWarning` (ADR-022 §2). > §3.2 notifications posted before the tag and cited by comment ID in the checklist (views-pipeline-core @@ -40,8 +66,10 @@ > dependency guard handles both pyproject layouts, and the doc guard scans fenced code in the section > being cut with real line numbers. Residuals stated in the docstrings. Published 21:27 UTC: publish > run 35396893934 ran the suite (993 passed, 1 skipped) before the upload; PyPI 2.0.0 declares numpy, -> scipy and the `frames` extra only. The library half of epic #66 is done; S9 and S10 (consumer pins) -> remain. Count unchanged. +> scipy and the `frames` extra only. The library half of epic #66 is done. **S10 done 2026-09-19:** +> views-pipeline-core pins `>=2.0.0,<3.0.0` (their #523), the tripwire is gone, and their WandB mean +> now skips `nan` (D-01's live residual closed on their side). S9 (views-reporting) is a check, not a +> change: their `main` already admits 2.0.0. Count unchanged. > **2026-09-18 — story S7 (#63) closed C-40 and Cluster A's second residue.** `to_dataframe()`, its > DataFrame helper and the three dead factories are gone; the `dataframe` extra is deleted and @@ -254,14 +282,14 @@ Root-cause groupings (added 2026-06-26; expanded then largely closed 2026-08-02 ### C-36 — ADR-022's release gates are unenforced prose - **Tier:** 2 (High) -- **Description:** ADR-022 §7 names a five-item **release checklist** as the policy's enforcement point, "worked through before any `poetry publish`". Nothing requires it to be worked through. It is a markdown list inside an ADR: no CI job, no template, no artifact that records it happened. The same is true of §3.2, which requires that "downstream consumers known to be affected must be **notified before** the release is cut, not after" — there is no place a notification is recorded, so its absence is indistinguishable from its presence. On the very first release governed by this policy (0.5.0, PR #45), the code review found the checklist had not been worked through anywhere, no `CHANGELOG.md` existed at all, and no committed artifact evidenced consumer notification — despite ADR-022 §6 stating outright that "the 0.5.0 release notes **must** retrospectively record the 0.4.0 removals as breaking". +- **Description:** ADR-022 §7 names a **release checklist** (five items until 2026-09-19, six since — the sixth asks about stale dependency caps, #91) as the policy's enforcement point, "worked through before any `poetry publish`". Nothing requires it to be worked through. It is a markdown list inside an ADR: no CI job, no template, no artifact that records it happened. The same is true of §3.2, which requires that "downstream consumers known to be affected must be **notified before** the release is cut, not after" — there is no place a notification is recorded, so its absence is indistinguishable from its presence. On the very first release governed by this policy (0.5.0, PR #45), the code review found the checklist had not been worked through anywhere, no `CHANGELOG.md` existed at all, and no committed artifact evidenced consumer notification — despite ADR-022 §6 stating outright that "the 0.5.0 release notes **must** retrospectively record the 0.4.0 removals as breaking". - **Why this is Tier 2, not Tier 3:** the failure it guards against has already happened and was expensive. On 2026-04-03, `EvaluationManager` and `PandasAdapter` were removed with no notice and **all six downstream integration tests in views-pipeline-core crashed with `ModuleNotFoundError`** (C-13). ADR-022 exists solely to stop that recurring. C-13's own closing rationale was that a register entry "has no enforcement point and will re-fire on every release" — and the policy written to supply that enforcement point supplied prose instead. The gap is in the mitigation, which is worse than a gap in a symptom. - **Trigger:** (a) The next release that changes behaviour for a known consumer without that consumer being told first (§3.2) — nothing records that a notification happened. (b) A release cut from a branch no pull request ever ran on: the guards run inside the suite they gate, so a neutered `publish_package.yml` on such a branch is not caught. (c) A `## [version]` section that exists but whose checklist was not worked through — the guard sees the heading, a real date and non-empty content, not the checklist. - **Location:** `documentation/ADRs/022_evolution_and_stability.md` §3.2 (notification) and §7 (the checklist); `.github/workflows/publish_package.yml`; `tests/test_falsification_extras_actually_installed.py` (`TestPublishGateIsReal`); `tests/test_documentation_contracts.py` (`TestChangelogCoversTheDeclaredVersion`); `CHANGELOG.md` - **Mechanical half (added 2026-09-16, repo-assimilation; closed 2026-09-17).** Until #68, the only automated gate between a GitHub release event and PyPI was "the version is higher than the last one on PyPI": `publish_package.yml` ran no tests and did not depend on `run_pytest.yml`, so a release cut from a red commit would have published. Independently checkable and independently fixable, separate from the checklist and notification halves. - **Source:** code-review of PR #45 (2026-08-02), findings R1-a, R1-b and F5; ungated publish workflow merged 2026-09-16 (repo-assimilation) - **Resolved in part (2026-09-17, epic #66, story #68):** the **mechanical half** is closed. `publish_package.yml` now `needs:` a job that runs `run_pytest.yml` by reference (`workflow_call`), so the release gate *is* the pull-request suite — install with all extras, extras verified, pytest, `validate_docs.sh` — and a red suite means no upload. `TestPublishGateIsReal` holds **every** file under `.github/workflows/` to a known-good text and the set of files to a known list (an allowlist; two review passes and two guard audits showed anything narrower loses — a parser-based denylist to every YAML feature it does not model, including `continue-on-error` on the pytest step or job, and a grep for publish commands to `uv publish`, `hatch publish`, a folded scalar, a composite action or a Makefile target). `TestChangelogCoversTheDeclaredVersion` fails when the declared version has no `## [x.y.z] — YYYY-MM-DD` section outside a fence or comment, with a real date and non-empty content. Observed red on injected defects: either workflow edited in any significant way (`if:`, `continue-on-error` on the pytest step or the job, `|| true`, `needs:` removed, `ref:` on checkout, `workflow_call` removed or moved under `jobs:`), a second workflow that publishes by any tool or extension, a composite action, a Makefile target reached from an existing workflow, version `9.9.9`, heading only inside a fence or comment, unclosed fence or comment, empty, heading-only or rule-only section, duplicate heading, impossible or future date. -- **Mitigation path:** Three open halves. *Notification* (§3.2): maintain a consumers list in-repo and require a linked notification issue per breaking release, or narrow §3.2 to something checkable and stop asserting a gate that cannot be verified — **asserting an unverifiable gate is worse than asserting none**, because it reads as satisfied. *Branch bypass*: GitHub-native — a protected `environment:` with required reviewers on the publish job, or restricting the release event to protected tags — so the guards' blind spot is closed outside the suite. *Checklist content*: the two shipped sections already share a shape (five `- [x] **Does …**` items); asserting that shape in the CHANGELOG guard is cheap if wanted. *Unheld call-outs* (third guard audit, 2026-09-17): the held workflow text calls `poetry run pytest tests/` and `bash documentation/validate_docs.sh`, and neither `tests/conftest.py` (does not exist today) nor `validate_docs.sh` is held to text — a change to either alters what CI runs without a red build. `[build-system]` in `pyproject.toml` *is* held, because it runs with the publish token in scope. The other two are reviewable diffs on any PR and are left to review. +- **Mitigation path:** Three open halves. *Notification* (§3.2): maintain a consumers list in-repo and require a linked notification issue per breaking release, or narrow §3.2 to something checkable and stop asserting a gate that cannot be verified — **asserting an unverifiable gate is worse than asserting none**, because it reads as satisfied. *Branch bypass*: GitHub-native — a protected `environment:` with required reviewers on the publish job, or restricting the release event to protected tags — so the guards' blind spot is closed outside the suite. *Checklist content*: the shipped sections share a shape (`- [x] **Does …**` items, five per section until 2.0.0 and six from 2.1.0 — a guard would have to count against ADR-022 §7's current list, not a literal); asserting that shape in the CHANGELOG guard is cheap if wanted. *Unheld call-outs* (third guard audit, 2026-09-17): the held workflow text calls `poetry run pytest tests/` and `bash documentation/validate_docs.sh`, and neither `tests/conftest.py` (does not exist today) nor `validate_docs.sh` is held to text — a change to either alters what CI runs without a red build. `[build-system]` in `pyproject.toml` *is* held, because it runs with the publish token in scope. The other two are reviewable diffs on any PR and are left to review. - **Note:** Distinct from C-34, which is about documentation *claims* drifting from *code*. This is about a documented *process* having no enforcement — the claim is not false, it is simply not binding on anyone. Cross-refs: C-13 (closed; this entry is the gap in its fix), C-24 (the cross-repo emit contract, which §7's fifth checklist item governs and which has the same no-enforcement problem). - **Notification half, first exercised as written (2026-09-18, release 1.1.0):** the §3.2 comments were posted on views-pipeline-core#512 and views-reporting#289 before the tag and are cited by comment ID in the 1.1.0 checklist — the first release to do so (0.5.0 and 1.0.0 both ticked the box with the notification scheduled after the tag, which the 1.1.0 release review caught in the draft). Still unenforced: nothing checks that the cited comment exists or preceded the tag; the mitigation path above stands. @@ -286,10 +314,11 @@ Root-cause groupings (added 2026-06-26; expanded then largely closed 2026-08-02 - **Description:** On 2026-08-02 the `views-frames` floor was raised from `^1.4` to `>=1.10.2,<2` and merged to `development` and `main`. **`views-reporting` consumes this repo by git branch**, not by version (`views-evaluation = { git = "...", branch = "development" }` in its `[tool.uv.sources]`), with no release, no version bump, and no opportunity for it to opt in. **Nobody checked the consumer's resolution before tightening the constraint** — that is the whole entry. Three separate attempts to write down what that consumer's lock actually held were each wrong, and the third went stale 62 seconds after it was written when the consumer committed, pushed and tagged a release. The correct conclusion is not a better narration of someone else's lockfile: it is that a constraint change here must not be made on an assumption about resolution there, and that a consumer tracking a branch receives such changes with none of a release's gates. - **The sharper case.** A `[tool.uv.sources]` override drops only the *first-party* specifier; it collapses the package's candidate set graph-wide, so any **other** package requiring an excluded version makes the whole resolve unsatisfiable. Widening one repo's pin can therefore break a *different* repo's resolve, and widening must proceed in dependency order. - **Why this is not C-35:** C-35 is about not exercising a consumer's *call path*. This is about not checking a consumer's *dependency resolution*. A library can be perfectly compatible at the API level and still be unresolvable, and no test in either repo would show it — the failure appears at install time, in their CI, after our merge. -- **Trigger:** The next time any constraint in `pyproject.toml` is narrowed — a raised floor, a lowered ceiling, a new required dependency — while any consumer resolves this repo from a branch rather than a published version. Branch-tracking makes every merge a release for that consumer, without any of a release's gates. -- **Location:** `pyproject.toml` (dependency constraints); `views-reporting/pyproject.toml` (formerly the `[tool.uv.sources]` branch-tracking entry; **as of 2026-09-16 it declares `views-evaluation[frames]>=1.0.0,<2.0.0`**, a plain version range) +- **Trigger:** The next time any constraint in `pyproject.toml` is narrowed — a raised floor, a lowered ceiling, a new required dependency — while any consumer resolves this repo from a branch rather than a published version; **and (added 2026-09-19, #91) whenever a dependency this repo caps publishes a release outside the declared range** — the ADR-022 §7 checklist now asks at every release, because a month passed between views-frames 2.0.0 and anyone noticing the `<2` cap. Branch-tracking makes every merge a release for that consumer, without any of a release's gates. +- **Location:** `pyproject.toml` (dependency constraints); `views-reporting/pyproject.toml` (formerly the `[tool.uv.sources]` branch-tracking entry; a plain version range since 2026-09-16 — its value is theirs and moves; read it there, not here) - **Source:** falsification audit round 3 (2026-08-02); consumer state re-checked 2026-09-16 (repo-assimilation) - **State update (2026-09-16):** views-reporting has executed this entry's cheapest mitigation — it now pins a published version range rather than a branch, so this repo's merges no longer reach it un-gated. The trigger remains valid for any *other* consumer that resolves from a branch, and the sharper resolver-collapse case above is unchanged. No branch-tracking consumer is known at the time of writing; that is an observation, not a guarantee (the 2026-08-02 narration failures above are the reason this entry does not describe consumer state in more detail). Inward-facing counterpart: C-41. +- **The mirror case (2026-09-19, #91):** a constraint can also be too *narrow* for too long. The `frames` extra capped views-frames `<2` for a month after views-frames 2.0.0 shipped, and the cap travels through the extra into every environment that requests it — an environment we do not own. Found by views-pipeline-core's range review, not ours; and their report's premise that our cap was the *only* bound in play was itself wrong (the consumer concerned also declares its own `<2.0.0`), which is the C-38 narration hazard in the other direction. Widened to `<3` on measurement (suite green on 1.10.2, 1.11.0 and 2.0.0; cross-major save/load round trip); `TestViewsFramesRange` and `TestViewsFramesSurface` pin the range, the one-name import surface and `FrameMetadata`'s behaviour, and CI runs the whole suite on the floor and asserts the resolved version is the 2.x end. The lesson is the same as this entry's: a dependency constraint in this repo is a decision about a consumer's environment, in both directions. - **Mitigation path:** Cheapest durable fix is to remove the reason for branch-tracking: now that the emit is published, views-reporting can revert to a plain version pin, at which point our merges stop reaching it un-gated. Failing that, treat narrowing a constraint as a breaking change under ADR-022 §3 and check the locks of known branch-tracking consumers first — the same "execute the real consumer's path" rule as C-35, applied to resolution rather than to calls. - **Note:** ADR-022 §3.2 already requires notifying known consumers before a release. Branch-tracking sidesteps that entirely, because there is no release to notify about. See C-35, C-36. @@ -297,10 +326,11 @@ Root-cause groupings (added 2026-06-26; expanded then largely closed 2026-08-02 ### C-41 — CI resolves dependencies fresh each run, tests one interpreter of four declared, under a numpy 1.x ceiling -- **Tier:** 3 (Medium) — reproducibility and coverage gap that raises the cost of every dependency change and every green-build claim; no correctness impact today. -- **Description:** No `poetry.lock` is committed (the `.gitignore` comment leaves it optional and none exists), so each `poetry install --all-extras` in `run_pytest.yml` resolves anew, and "CI passed" does not name the versions it passed against. `pyproject.toml` declares `python = ">=3.11,<3.15"` while `run_pytest.yml` pins `python-version: "3.11"` — three of the four claimed interpreters are never exercised. `numpy = "^1.26.4"` caps at the last 1.x line while `scikit-learn` (`^1.6.0`), `scipy` (`^1.11`) and `views-frames` (`>=1.10.2,<2`) each admit every later minor release below 2.0, so a *minor* release of any of them that drops numpy 1.x support makes the resolve unsatisfiable, or — worse — resolves to a different combination between two runs with no diff to show it. The local environment used for the 2026-09-16 assimilation resolved to numpy 1.26.4, scipy 1.15.1, scikit-learn 1.7.2, pandas 1.5.3, views-frames 1.10.2; nothing records that CI resolved the same. +- **Tier:** 2 (High) — **re-tiered 2026-09-19 on measurement:** the numpy `<2.0.0` cap makes the published wheel unresolvable from wheels on CPython 3.13 and 3.14, both inside the declared `python = ">=3.11,<3.15"` (`pip install --dry-run --only-binary=:all: --python-version 3.13 views-evaluation==2.0.0` → `No matching distribution found for numpy<2.0.0,>=1.26.4`; 3.11 and 3.12 resolve). A declared interpreter on which the package cannot be installed is a consumer-facing failure, not a coverage gap. Was Tier 3. +- **Description:** No `poetry.lock` is committed (the `.gitignore` comment leaves it optional and none exists), so each `poetry install --all-extras` in `run_pytest.yml` resolves anew, and "CI passed" does not name the versions it passed against. `pyproject.toml` declares `python = ">=3.11,<3.15"` while `run_pytest.yml` pins `python-version: "3.11"` — three of the four claimed interpreters are never exercised. `numpy = "^1.26.4"` caps at the last 1.x line while `scikit-learn` (`^1.6.0`), `scipy` (`^1.11`) and `views-frames` (`>=1.10.2,<3` since 2026-09-19, #91 — a range that spans two majors, both run in CI; `run_pytest.yml`'s verify step prints and asserts which views-frames it resolved, a partial mitigation of this entry) each admit every later minor release below 2.0, so a *minor* release of any of them that drops numpy 1.x support makes the resolve unsatisfiable, or — worse — resolves to a different combination between two runs with no diff to show it. The local environment used for the 2026-09-16 assimilation resolved to numpy 1.26.4, scipy 1.15.1, scikit-learn 1.7.2, pandas 1.5.3, views-frames 1.10.2; nothing records that CI resolved the same. - **Trigger:** When scikit-learn, scipy or views-frames publishes a release that requires numpy ≥ 2, or when a maintainer adds Python 3.12+ to the CI matrix, or when a CI run goes red with no change to this repo — check what CI actually resolved, and whether it can be reproduced. -- **Location:** `pyproject.toml` (`numpy = "^1.26.4"`, `python = ">=3.11,<3.15"`, caret ranges on `scikit-learn` and `scipy` that admit future 1.x minors); `.github/workflows/run_pytest.yml` (`python-version: "3.11"`; `poetry install --all-extras` without a lock); absence of `poetry.lock` +- **Trigger fired, unnoticed (recorded 2026-09-19):** scipy 1.18.0 (2026-06-19) requires `numpy>=2.0.0` and `python>=3.12`. Under our numpy cap the resolver did not go red — it backtracked to scipy 1.17.x silently (CI run 35416248120 installed scipy 1.17.1 and printed nothing about it), which is the outcome neither clause of the description foresaw: not an unsatisfiable resolve but a frozen effective scipy ceiling. Found by the 2.1.0 release review, three months late. The numpy cap is therefore **stale, not deliberate**: it has never been measured under numpy 2 and it excludes two declared interpreters. Tracked as **#93** (measure numpy 2.x, lift or narrow `python`). +- **Location:** `pyproject.toml` (`numpy = "^1.26.4"`, `python = ">=3.11,<3.15"`, `scipy = "^1.11"` whose effective ceiling is now set by the numpy cap, `views-frames >=1.10.2,<3`); `.github/workflows/run_pytest.yml` (`python-version: "3.11"`; `poetry install --all-extras` without a lock); absence of `poetry.lock` - **Source:** repo-assimilation (2026-09-16) - **Mitigation path:** Commit `poetry.lock` (or record the resolved set in the CI log as an artifact) so a green build names its versions; add the declared interpreters to the CI matrix or narrow the declared range to what is tested; decide deliberately whether the numpy 1.x ceiling is a constraint or an oversight, and either lift it with a test run under numpy 2 or document why it stays. - **Note:** Inward-facing counterpart of C-38 — that entry is about consumers' resolution being unchecked; this is about this repo's own resolution being unrecorded. Same evidence gap as C-35: "it passed" without "against what". **2026-09-18:** the resolve shrank — scikit-learn left the runtime dependencies (C-05, #64) and is now dev-only; the parity suite's oracle is whatever scikit-learn CI resolves, which this entry's lockfile point now also covers. @@ -339,7 +369,7 @@ Maintainer decisions taken on a register concern, recorded here so the reasoning - **Options weighed:** (A) `nan`, warning suppressed — chosen; (B) raise — rejected on the R2 evidence (aborts every metric and schema for a routine data condition; tried on Pearson and reversed the same day); (C) keep `0.0`, warning suppressed — rejected because it codifies a wrong number into the evaluation-of-record and silently pulls the `mean` row toward zero by an amount set by the data, not the model (the C-02 shape). - **Consequence (ADR-022 §3):** a behaviour change for any evaluation whose data contains an empty classification group — `nan` for the group, `mean` row over the rest, where before it was `0.0` and a lower mean. Affected at the time of decision: `views-models/ensembles/rusty_bucket` (the one live config requesting `AP`) and views-reporting's canonical classification cell. Both to be named in the release notes of the release that carries the switch (S6, 1.1.0). - **Ships with:** #64. **Shipped:** 1.1.0 published to PyPI 2026-09-18 with the change in its notes; this decision's consequence is live. -- **Residual, now live (found by the code review of #64, 2026-09-18):** C-22's — and the direct `to_dict()` reader it warned about exists: views-pipeline-core averages group values for its WandB scalars with a mean that skips `None` but not `nan` (`views_pipeline_core/modules/wandb/utils.py`, observed on their `development` branch). One empty group makes their `AP_mean` scalar `nan`. The fix is theirs (`nanmean`), named in the release notes, and C-22's trigger is widened to cover it. Also, until 2.0.0: `to_dataframe()` dropped an all-`nan` column (C-40), which applied to `AP` too; the method was deprecated in 1.1.0 and removed in 2.0.0 (S7, #63; C-40 closed 2026-09-18). +- **Residual, now live (found by the code review of #64, 2026-09-18):** C-22's — and the direct `to_dict()` reader it warned about exists: views-pipeline-core averages group values for its WandB scalars with a mean that skips `None` but not `nan` (`views_pipeline_core/modules/wandb/utils.py`, observed on their `development` branch). One empty group makes their `AP_mean` scalar `nan`. The fix is theirs (`nanmean`), named in the release notes, and C-22's trigger is widened to cover it. **Resolved on their side 2026-09-19 (their PR #523, `e65f705`, their C-329; ships in their 3.3.0):** reproduced on 2.0.0, then fixed — nanmean semantics, `inf` still a number, an all-`nan` metric omitted. Their consumer assessment of 1.1.0/2.0.0 (tracker #73) found no defect on this side; nothing their pipeline sends is on the 1.1.0 refused-inputs list. Also, until 2.0.0: `to_dataframe()` dropped an all-`nan` column (C-40), which applied to `AP` too; the method was deprecated in 1.1.0 and removed in 2.0.0 (S7, #63; C-40 closed 2026-09-18). - **Versioning:** filed MINOR on the ADR-022 §1 argument that the `0.0` was never documented behaviour of this library; recorded in ADR-015 R9 so the S6 checklist can answer rule 5 against it. --- diff --git a/tests/test_falsification_extras_actually_installed.py b/tests/test_falsification_extras_actually_installed.py index fe11048..2dee4d9 100644 --- a/tests/test_falsification_extras_actually_installed.py +++ b/tests/test_falsification_extras_actually_installed.py @@ -24,6 +24,7 @@ dark, or the proof that it has not is no longer a proof. """ import re +import sys from pathlib import Path import pytest @@ -107,6 +108,198 @@ def _strip_comment(s): return "".join(out).rstrip() +# Every way an executed command's failure can be prevented from failing the job. Each +# was confirmed to defeat an earlier version of a guard here. Module-level so every +# step-reading guard applies the same list (the floor-step guard had two of these). +_SUPPRESSED = ( + r"\|\|", # `|| true`, `|| :`, `|| echo skip` — any fallback + r"^\s*set \+(e|o\s+errexit)", # errexit disabled + r"^\s*if\s|;\s*then\b", # wrapped in a conditional that swallows status + r"&\s*$", # backgrounded + r";\s*exit\s+0", # status discarded + r"^\s*exit\s+0\s*$", # status discarded, on its own line + r";\s*(true|:)\s*$", # status replaced + r"&\s*wait\s*$", # backgrounded, then waited on — `wait` exits 0 + # Nine more, each proven to exit 0 under `bash -e` by the 2.1.0 guard audit: + r"\|\s*\w", # piped into anything: no pipefail, the pipe's status wins + r"\bset\s+\+\S*e\b", # `set +e`, `set +xe`, `set -e; set +e` — errexit off, anywhere on the line + r"^\s*exit\b", # `exit 0;`, `exit 00`, bare `exit` — own-line exit of any spelling + r"^\s*(echo|printf)\b.*\$\(", # `echo $(cmd)`: the substitution's status is discarded + r"\bwhile\s+!", # `while ! cmd; do break; done` + r"&\s*\)", # `(cmd &)`: backgrounded inside a subshell +) + + +def _views_frames_specifier(): + """The declared views-frames version specifier, from either pyproject layout — + PEP 621 FIRST, as `_extras_table` reads it and as poetry >= 2 does, so a stale + poetry entry beside an authoritative `[project]` table cannot be the one read + (guard audit, 2026-09-19). Rejects an environment marker on the entry: a + `python = ">=3.12"` key or a `; python_version` marker leaves the extra unresolved + on the 3.11 runner with the specifier text unchanged.""" + import tomllib + from packaging.requirements import Requirement + data = tomllib.loads(PYPROJECT.read_text(encoding="utf-8")) + project = data.get("project", {}) + + def canonical(name): + return re.sub(r"[-_.]+", "-", name).lower() + + # EVERY declaration, then exactly one: a second views-frames entry (another extra, + # a hard `[project] dependencies` line, a duplicate in one list) narrows the range + # the resolver applies while a first-match reader kept returning the first (guard + # audit, 2026-09-19). Poetry-table keys are canonicalised as poetry does. + found = [] + for entry in project.get("dependencies", []) + sum(project.get("optional-dependencies", {}).values(), []): + req = Requirement(entry) # PEP 508, parenthesised form included — what poetry 2 writes + if canonical(req.name) == "views-frames": + assert req.marker is None, f"views-frames carries an environment marker: {entry!r}" + found.append(str(req.specifier)) + for key, value in data.get("tool", {}).get("poetry", {}).get("dependencies", {}).items(): + if canonical(key) != "views-frames": + continue + if isinstance(value, dict): + assert set(value) <= {"version", "optional"}, f"views-frames poetry entry carries extra keys: {sorted(value)}" + found.append(value["version"].replace(" ", "")) + else: + found.append(str(value).replace(" ", "")) + assert len(found) == 1, f"views-frames must be declared exactly once; found {found}" + return found[0] + + +class TestViewsFramesRange: + """The `frames` extra admits two views-frames majors (`>=1.10.2,<3`, #91), measured + on both ends. Two static pins, hosted here because this module never import-skips: + the declared range itself, and the one-name import surface that justifies it. The + behavioural half is `tests/test_metric_frame.py::TestViewsFramesSurface`.""" + + def test_the_declared_range_is_the_measured_one(self): + """Compared as specifier sets, so `<3,>=1.10.2` or a space is not a false alarm.""" + from packaging.specifiers import SpecifierSet + spec = _views_frames_specifier() + assert SpecifierSet(spec) == SpecifierSet(">=1.10.2,<3"), ( + f"views-frames is declared {spec!r}; the measured range is >=1.10.2,<3 (#91). " + f"Widen or narrow only with the suite green on both ends of the new range." + ) + + def test_the_ci_floor_step_derives_the_floor_from_pyproject(self): + """The floor step must read the floor through the one specifier reader, carry no + version literal of its own (a hand-copied `1.10.2` is a second source a floor + raise can miss), and run install → check → import proof → whole suite in that + order with no way to suppress a failure. Deliberately strict about spelling: + a heredoc derivation or `--no-deps` on another line is a red build until this + guard is updated with it, because the held workflow text would be too.""" + steps = _workflow_steps(WORKFLOW.read_text(encoding="utf-8")) + floor = [s for s in steps if "floor" in s["name"].lower()] + assert len(floor) == 1, [s["name"] for s in steps] + lines = _significant_lines(floor[0]["run"]) + run = "\n".join(lines) + # ONE reader: the step must obtain the floor by calling `_views_frames_specifier` + # from this module, not by a second inline parser (a second reader shipped with + # measured divergences from the first; release review 2026-09-19). A literal + # carrying the right tokens in a comment passed a token check, so the check is + # on the call, and the helper itself is exercised on a fixture below. + # Structural, in order: derivation → install → check → import proof → pytest, + # each identified by content, FLOOR assigned exactly once. A substring check + # was satisfied by the right tokens in the wrong place — an echo, a comment + # inside the `-c` quotes, an `export FLOOR=1.10.2` after the real derivation, + # an import that ran before the install (guard audit, 2026-09-19). + assignments = [line for line in lines if re.search(r"\bFLOOR=", line)] + assert len(assignments) == 1 and lines.index(assignments[0]) == 1, assignments # right after `set -e` + derivation = assignments[0] + assert derivation.startswith("FLOOR=$(poetry run python -c "), derivation + assert "_views_frames_specifier()" in derivation and "print(min(" in derivation, derivation + assert not re.search(r"print\(\s*['\"]", derivation), "the derivation prints a literal, not the helper's result" + + def index_of(predicate, what): + hits = [i for i, line in enumerate(lines) if predicate(line)] + assert len(hits) == 1, f"{what}: expected exactly one line, got {hits}" + return hits[0] + + i_install = index_of(lambda line: line == 'poetry run pip install --quiet "views-frames==$FLOOR" --no-deps', "install with --no-deps") + i_check = index_of(lambda line: line == "poetry run pip check", "pip check") + i_proof = index_of(lambda line: line.startswith('poetry run python -c "import views_frames') + and "assert m.version('views-frames') == sys.argv[1]" in line and line.endswith('"$FLOOR"'), + "import proof pinned to FLOOR") + i_pytest = index_of(lambda line: line == "poetry run pytest tests/ -q", "whole suite") + assert 1 < i_install < i_check < i_proof < i_pytest == len(lines) - 1, [i_install, i_check, i_proof, i_pytest, len(lines)] + assert not re.search(r"views-frames==\d", run), "the floor step hard-codes a version" + for pattern in _SUPPRESSED: + assert not re.search(pattern, run, re.M), f"a suppressed exit in the floor step: {pattern!r}" + assert not floor[0]["continue_on_error"], "the floor step carries `continue-on-error:` or `if:`" + assert "PYTEST_ADDOPTS" not in WORKFLOW.read_text(encoding="utf-8"), "an env var can narrow the suite behind the exact command" + + def test_the_specifier_helper_reads_every_layout_it_claims_to(self, monkeypatch, tmp_path): + """The helper on fixtures: the poetry dict, the poetry string, and the PEP 508 + parenthesised form poetry 2 writes into `[project.optional-dependencies]` — + which a regex reader turned into `'(>=1.10.2'` (release review, 2026-09-19). + And the floor the CI step derives from it.""" + cases = { + 'views-frames = {version = ">=9.9.9,<10", optional = true}\n[tool.poetry.extras]\nframes = ["views-frames"]\n': + ">=9.9.9,<10", + '[tool.poetry.dependencies]\nviews-frames = ">= 9.9.9, <10"\n': ">=9.9.9,<10", + '[project]\nname = "x"\n[project.optional-dependencies]\nframes = ["views-frames (>=9.9.9,<10)"]\n': "<10,>=9.9.9", + '[project]\nname = "x"\n[project.optional-dependencies]\nframes = ["views-frames-reconcile>=1", "views-frames>=9.9.9,<10"]\n': "<10,>=9.9.9", + } + from packaging.specifiers import SpecifierSet + for text, expected in cases.items(): + if not text.startswith("["): + text = "[tool.poetry.dependencies]\n" + text + p = tmp_path / "pyproject.toml" + p.write_text(text, encoding="utf-8") + monkeypatch.setattr(sys.modules[__name__], "PYPROJECT", p) + assert SpecifierSet(_views_frames_specifier()) == SpecifierSet(expected), text + floor = min(str(s.version) for s in SpecifierSet(_views_frames_specifier()) if s.operator == ">=") + assert floor == "9.9.9" + p.write_text('[project]\nname = "x"\ndependencies = ["views-frames>=9.9.9,<10; python_version >= \'3.12\'"]\n', encoding="utf-8") + with pytest.raises(AssertionError, match="environment marker"): + _views_frames_specifier() + + def test_the_only_name_imported_from_views_frames_is_frame_metadata(self): + """AST over every package file: `from views_frames import X`, `import + views_frames[.sub] as y`, and `views_frames.X` attribute access may name only + FrameMetadata. A second name is a second thing that can change between majors + and is a red build until measured. (`import views_frames as vf; vf.X` was + invisible to the first version, and `getattr(views_frames, "X")` and + `_vf = views_frames; _vf.X` to the second — release review and guard audit, + 2026-09-19. Import machinery — `import_module("views_frames").X` — is forbidden + package-wide by `TestLevelZeroImportPurity`.)""" + import ast + import views_evaluation + names = set() + for path in Path(views_evaluation.__file__).parent.rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8")) + aliases = {"views_frames"} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for a in node.names: + if a.name.split(".")[0] == "views_frames": + names.add(a.name) if "." in a.name else None + aliases.add(a.asname or a.name.split(".")[0]) + # Re-aliasing by assignment (`_vf = views_frames`), to a fixed point. + grew = True + while grew: + grew = False + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Name) and node.value.id in aliases: + for t in node.targets: + if isinstance(t, ast.Name) and t.id not in aliases: + aliases.add(t.id) + grew = True + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and (node.module or "").split(".")[0] == "views_frames": + names |= {f"{node.module}.{a.name}" for a in node.names} + elif isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id in aliases: + names.add(f"views_frames.{node.attr}") + elif (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "getattr" + and node.args and isinstance(node.args[0], ast.Name) and node.args[0].id in aliases): + # `getattr(views_frames, "X")`: a literal is a name; anything else is + # a name this guard cannot read, which is as bad. + attr = node.args[1] if len(node.args) > 1 else None + names.add(f"views_frames.{attr.value}" if isinstance(attr, ast.Constant) else "views_frames.") + assert names == {"views_frames.FrameMetadata"}, sorted(names) + + def _commands(run): """Executable command lines from a run block, with inert ones dropped. @@ -215,17 +408,7 @@ def _index(pred): # the standard "unbreak the flaky step" edit, and it leaves the guard green # while disabling the check entirely. That is register C-37's vacuity shape # reproduced inside the test written to close it. - # Every way an executed command's failure can be prevented from failing the job. - # Each was confirmed to defeat an earlier version of this guard. - SUPPRESSED = ( - r"\|\|", # `|| true`, `|| :`, `|| echo skip` — any fallback - r"^\s*set \+(e|o\s+errexit)", # errexit disabled - r"^\s*if\s|;\s*then\b", # wrapped in a conditional that swallows status - r"&\s*$", # backgrounded - r";\s*exit\s+0", # status discarded - r";\s*(true|:)\s*$", # status replaced - r"&\s*wait\s*$", # backgrounded, then waited on — `wait` exits 0 - ) + SUPPRESSED = _SUPPRESSED # This list is defence-in-depth: TestPublishGateIsReal holds every workflow file # to a known-good text, which is what catches an evasion this list does not name. @@ -466,11 +649,20 @@ class TestPublishGateIsReal: PY poetry install --all-extras - name: Verify optional extras are installed - run: poetry run python -c "import views_frames" + run: poetry run python -c "import views_frames, importlib.metadata as m; from tests.test_falsification_extras_actually_installed import _views_frames_specifier; from packaging.specifiers import SpecifierSet; v = m.version('views-frames'); spec = SpecifierSet(_views_frames_specifier()); floor = min(str(s.version) for s in spec if s.operator == '>='); print('resolved views-frames', v, 'declared', spec, 'floor', floor); assert spec.contains(v), 'resolved outside the declared range'; assert v.split('.')[0] != floor.split('.')[0], 'the resolver picked the floor major; the other end of the range would go untested'" - name: Run tests run: | set -e poetry run pytest tests/ + - name: Run the suite on the views-frames floor + run: | + set -e + FLOOR=$(poetry run python -c "from tests.test_falsification_extras_actually_installed import _views_frames_specifier; from packaging.specifiers import SpecifierSet; print(min(str(s.version) for s in SpecifierSet(_views_frames_specifier()) if s.operator == '>='))") + echo "views-frames floor from pyproject.toml: $FLOOR" + poetry run pip install --quiet "views-frames==$FLOOR" --no-deps + poetry run pip check + poetry run python -c "import views_frames, importlib.metadata as m, sys; from views_frames import FrameMetadata; assert m.version('views-frames') == sys.argv[1], m.version('views-frames')" "$FLOOR" + poetry run pytest tests/ -q - name: Validate documentation consistency run: bash documentation/validate_docs.sh """, diff --git a/tests/test_metric_frame.py b/tests/test_metric_frame.py index 71e6264..a16f3da 100644 --- a/tests/test_metric_frame.py +++ b/tests/test_metric_frame.py @@ -969,3 +969,52 @@ def test_submodule_relative_gitdir_resolves_against_the_git_file(self, tmp_path, monkeypatch.setattr(mf_mod, "__file__", str(_fake_package_file(repo))) assert mf_mod._source_git_sha() == "1234567" + + + +class TestViewsFramesSurface: + """The `frames` extra admits two views-frames majors (`>=1.10.2,<3`, #91). That is + safe only because this package's runtime surface into views-frames is one name, + `FrameMetadata`, used as a plain dataclass: six keyword fields, `to_dict` (omitting + None), `from_dict` (ignoring keys it does not own, which `MetricFrame.load` relies + on), default construction. views-frames 2.0.0's ADR-028 changes (a frame's index + type, read-only `.values`, `map_estimate` refusals) never reach it. Measured + 2026-09-19: the full suite (at its 2.0.0 count, before these tests) green on 1.10.2, + 1.11.0 and 2.0.0, and a frame saved + under either major loads under the other. The declared range and the one-name + import surface are pinned in tests/test_falsification_extras_actually_installed.py + (outside this module's import-skip); this class pins the BEHAVIOUR, against + whichever views-frames is installed.""" + + def test_installed_views_frames_is_inside_the_declared_range(self): + """Not `major in (1, 2)`: that is a third copy of the ceiling and admits a + below-floor release (1.9.0 passed it). The declared specifier is the one source.""" + import importlib.metadata as md + from packaging.specifiers import SpecifierSet + from tests.test_falsification_extras_actually_installed import _views_frames_specifier + installed = md.version("views-frames") + assert SpecifierSet(_views_frames_specifier()).contains(installed, prereleases=True), ( + f"views-frames {installed} is outside the declared range; measure before widening" + ) + + def test_frame_metadata_offers_the_surface_this_package_uses(self): + """Behavioural, against the installed major: the six keyword fields + `to_metric_frame` passes; default construction (reached through + `MetricFrame(..., metadata=None)` → `default_factory=FrameMetadata`); `to_dict` + omitting None; `from_dict` ignoring the keys `MetricFrameMetadata.to_dict()` + adds beside the generic ones (`schema_version`, `scoring_code_version`, + `evaluation_timestamp`) — which is what `MetricFrame.load()` hands it — and an + int timestamp surviving the round trip as an int, not a float that compares equal.""" + fm = FrameMetadata(model="m", run_type="test", timestamp=1758153600, seed=7, run_id="r", data_version="v") + assert FrameMetadata.from_dict(fm.to_dict()) == fm + back = FrameMetadata.from_dict({**fm.to_dict(), "schema_version": "1.0.0", "scoring_code_version": "2.1.0", + "evaluation_timestamp": "2026-09-19T00:00:00"}) + assert back == fm, "from_dict must ignore the MetricFrameMetadata-owned keys load() passes through" + assert type(back.timestamp) is int + assert FrameMetadata().to_dict() == {}, "to_dict must omit None so metadata.json keeps its shape" + assert FrameMetadata(model="m").to_dict() == {"model": "m"} + # None, not falsy: `seed=0` and `timestamp=0` are values and must survive the + # trip, or a run seeded 0 would load with `seed=None` (guard audit, 2026-09-19). + zero = FrameMetadata(model="m", run_type="t", timestamp=0, seed=0, run_id="", data_version="") + assert FrameMetadata.from_dict(zero.to_dict()) == zero + assert zero.to_dict()["seed"] == 0 and zero.to_dict()["timestamp"] == 0