diff --git a/.github/workflows/run_pytest.yml b/.github/workflows/run_pytest.yml index 1e2d99f..7e7b354 100644 --- a/.github/workflows/run_pytest.yml +++ b/.github/workflows/run_pytest.yml @@ -27,36 +27,44 @@ jobs: # is normal. The consequence was that CI verified strictly LESS than a developer's # laptop, and precisely on the guards that matter most across a repo boundary. # - # Measured 2026-08-03 (402 collected in all three runs, so the delta is entirely - # skips): - # local, all siblings present 362 passed / 40 xfailed / 0 skipped - # CI, bare checkout 347 passed / 38 xfailed / 17 skipped - # CI, with the checkout below 348 passed / 38 xfailed / 16 skipped + # WHICH siblings are checked out is NOT decided here. It is declared in + # `tests/conftest.py::SIBLINGS`, and `tests/test_ci_sibling_coverage.py` fails if + # this file and that declaration disagree — in either direction. ADR-016. # - # So this recovers exactly ONE test — the cross-seam consumer-name check for CRAF'd. - # That is worth having (it is the §4.1a pin, whose failure mode is a delivery nobody - # can find) but it is one, not seventeen. The other sixteen are below. + # The prose that used to live here said views-appwrite was PRIVATE. It went public + # on 2026-08-08 and the comment did not, so seven checks stayed dark for no reason. + # That is why the fact now lives in a declaration carrying the date it was checked, + # rather than in a paragraph. # - # Only views-crafdapi is checked out, and the choice is deliberate: + # `ref: main` is deliberate on every sibling and is not a default. Without it + # `actions/checkout` takes the sibling's OWN default branch — which for + # views-appwrite is `development` — while ADR-014 §3 says the authority for a claim + # about another repository is that repository's `main`. Two checks in + # test_env_declaration read the working tree and a third demands reachability from + # `main`; pointed at `development` they would eventually demand contradictory things. # - # views-crafdapi PUBLIC, and its check reads source text -> enabled here. - # views-datafactory PUBLIC, but its 8 tests need the producer's raw GAUL - # parquets (`data/raw/gaul_admin/*.parquet`), which are NOT in - # its git repository. Checking it out converts an honest skip - # into a FileNotFoundError. Measured, not assumed — it was tried. - # views-appwrite PRIVATE. The registry-drift detectors live here and are the - # views-faoapi most valuable of the lot; enabling them needs a token, which - # is an operator decision. Register C-46 and C-57 both carry - # that as their standing residual, and it is still open. # `actions/checkout` refuses a path outside $GITHUB_WORKSPACE, so the siblings go # inside it and are pointed at by the declared env vars instead of the conventional # `../` location. `sibling_repo` checks $VIEWS_ first, precisely so the - # location can be declared rather than assumed. - - name: Checkout views-crafdapi (sibling) + # location can be declared rather than assumed. `_siblings/` specifically: it is + # excluded from ruff in pyproject.toml, and the Lint step reported 745 findings in + # someone else's test suite the first time a sibling landed outside it. + # + # No `token:` on any step below, and that is load-bearing rather than an omission: + # the default GITHUB_TOKEN is scoped to THIS repository, so a tokenless checkout of a + # private repository simply fails, loudly, at this step. + # + # An earlier version of this comment said that "verifies the `public` field". There is + # no such field: `public` was deleted from `SIBLINGS` on 2026-08-10 because nothing + # could verify it, and ADR-016 §4 now says so explicitly. A comment in THIS file + # asserting a fact about another repository that nothing checks is the founding + # incident of ADR-016 §3, and it had reappeared here. + - name: Checkout views-appwrite (sibling) uses: actions/checkout@v3 with: - repository: views-platform/views-crafdapi - path: _siblings/views-crafdapi + repository: views-platform/views-appwrite + ref: main + path: _siblings/views-appwrite fetch-depth: 0 - name: Set up Python @@ -80,7 +88,7 @@ jobs: - name: Run tests env: - VIEWS_CRAFDAPI: ${{ github.workspace }}/_siblings/views-crafdapi + VIEWS_APPWRITE: ${{ github.workspace }}/_siblings/views-appwrite run: | set -e poetry run pytest tests/ diff --git a/README.md b/README.md index 4f4cec9..361fbd4 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Requires **Python 3.11–3.14**. | Package | Version | Why | |---------|---------|-----| | `views-pipeline-core` | `>=3.0.0,<4.0.0` (with the `appwrite` extra) | The framework: lifecycle base classes, data loader, dataset container, Appwrite/datastore tools | -| `views-frames` | `>=1.10.2,<2` | The frame data contract — **the live delivery representation** since #126. pandas survives only in `contract/enrichment.py` (the build/verification path) | +| `views-frames` | `>=1.10.2,<2` | The frame data contract — **the live delivery representation** since #126. There is no pandas in this package at all since #90 | | `pyarrow` | `>=16.1.0,<17.0.0` | The wire's serialisation. **Pinned deliberately** — the CVE fix past 17 changes delivered bytes (register C-72) | | *dev group* | `pytest`, `ruff` | Not installed by `pip install views-postprocessing`; `poetry install` includes them | @@ -142,7 +142,6 @@ views-postprocessing/ │ ├── historical.py # the historical artifact, built pandas-free │ ├── gaul_lookup.py # the GAUL asset: path, version, one read │ ├── gaul_schema.py # the 9-column contract, declared as data - │ ├── enrichment.py # GaulLookupEnricher (build/verification path) │ ├── source_metadata.py # producer (datafactory) facts │ ├── store_metadata.py # prediction-store facts │ └── launch_config.py # the delivery mode the launcher must declare @@ -220,7 +219,7 @@ only `.py`; it now scans markdown too.)* |-----|----------------| | [`docs/architecture/role_and_seams.md`](docs/architecture/role_and_seams.md) | **Start here** — role vs the sibling repos + internal seams | | [`docs/ADRs/`](docs/ADRs/) | Architecture decisions (esp. ADR-011 mapper→lookup; ADR-012 ontology) | -| [`docs/CICs/`](docs/CICs/) | Class intent contracts (`UNFAOPostProcessorManager`, `GaulLookupEnricher`) | +| [`docs/CICs/`](docs/CICs/) | Class intent contracts (`UNFAOPostProcessorManager`) | | `reports/technical_risk_register.md` | Tracked risks — C-40 (the remaining pipeline-core inheritance), C-30/C-15 (delivery guards), C-43 (enrichment value verification) | --- diff --git a/docs/ADR-011_implementation_assessment.md b/docs/ADR-011_implementation_assessment.md index 83e3714..9259901 100644 --- a/docs/ADR-011_implementation_assessment.md +++ b/docs/ADR-011_implementation_assessment.md @@ -1,6 +1,14 @@ # ADR-011 Implementation Assessment **Last updated:** 2026-06-12 +> **This is a dated snapshot (2026-06-12), not a description of the code today.** It is +> kept as a record of what was known when ADR-011 was assessed. Several things it +> describes as current have since been removed — notably the `.env` credential borrow +> (killed by þing-01 #134; the environment is now declared and validated fail-loud in +> each partner's `appwrite_env.py`) and `self.ensemble_path_manager`, which stopped +> existing on 2026-08-05 when store construction moved off the manager class (register +> C-40). Read §8 in particular as history. + **Status:** Data prerequisites MET — see §10. Implementation unblocked on the data side; verification infrastructure (Appwrite, pipeline-core E2E) still required before switching the pipeline. --- diff --git a/docs/ADRs/000_use_of_adrs.md b/docs/ADRs/000_use_of_adrs.md index ee9b9b3..e30bd47 100644 --- a/docs/ADRs/000_use_of_adrs.md +++ b/docs/ADRs/000_use_of_adrs.md @@ -3,7 +3,7 @@ **Status:** Accepted **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase **Informed:** All contributors --- diff --git a/docs/ADRs/001_ontology_of_views_postprocessing.md b/docs/ADRs/001_ontology_of_views_postprocessing.md index 0a32b2e..df9a73d 100644 --- a/docs/ADRs/001_ontology_of_views_postprocessing.md +++ b/docs/ADRs/001_ontology_of_views_postprocessing.md @@ -3,7 +3,7 @@ **Status:** Superseded by [ADR-012](012_revised_ontology.md) (2026-06-27) **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase --- diff --git a/docs/ADRs/002_topology_and_dependency_rules.md b/docs/ADRs/002_topology_and_dependency_rules.md index ebf9a7a..83e4ff3 100644 --- a/docs/ADRs/002_topology_and_dependency_rules.md +++ b/docs/ADRs/002_topology_and_dependency_rules.md @@ -2,7 +2,7 @@ **Status:** Accepted — amended 2026-06-27 (see [Amendment](#amendment-2026-06-27)) **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase --- diff --git a/docs/ADRs/003_authority_of_declarations_over_inference.md b/docs/ADRs/003_authority_of_declarations_over_inference.md index cd2e868..ac664c3 100644 --- a/docs/ADRs/003_authority_of_declarations_over_inference.md +++ b/docs/ADRs/003_authority_of_declarations_over_inference.md @@ -3,7 +3,7 @@ **Status:** Accepted **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase --- diff --git a/docs/ADRs/005_testing_as_mandatory_critical_infrastructure.md b/docs/ADRs/005_testing_as_mandatory_critical_infrastructure.md index 534673e..0bb32c9 100644 --- a/docs/ADRs/005_testing_as_mandatory_critical_infrastructure.md +++ b/docs/ADRs/005_testing_as_mandatory_critical_infrastructure.md @@ -3,7 +3,7 @@ **Status:** Accepted **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase --- diff --git a/docs/ADRs/006_intent_contracts_for_non_trivial_classes.md b/docs/ADRs/006_intent_contracts_for_non_trivial_classes.md index 46827bc..5d11083 100644 --- a/docs/ADRs/006_intent_contracts_for_non_trivial_classes.md +++ b/docs/ADRs/006_intent_contracts_for_non_trivial_classes.md @@ -3,7 +3,7 @@ **Status:** Accepted **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase --- diff --git a/docs/ADRs/007_silicon_based_agents_as_untrusted_contributors.md b/docs/ADRs/007_silicon_based_agents_as_untrusted_contributors.md index bbf45ab..1703917 100644 --- a/docs/ADRs/007_silicon_based_agents_as_untrusted_contributors.md +++ b/docs/ADRs/007_silicon_based_agents_as_untrusted_contributors.md @@ -3,7 +3,7 @@ **Status:** Accepted **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase --- diff --git a/docs/ADRs/008_observability_and_explicit_failure.md b/docs/ADRs/008_observability_and_explicit_failure.md index c48583a..9d22e06 100644 --- a/docs/ADRs/008_observability_and_explicit_failure.md +++ b/docs/ADRs/008_observability_and_explicit_failure.md @@ -3,7 +3,7 @@ **Status:** Accepted **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase --- diff --git a/docs/ADRs/009_boundary_contracts_and_configuration_validation.md b/docs/ADRs/009_boundary_contracts_and_configuration_validation.md index 6e7c0d7..3668251 100644 --- a/docs/ADRs/009_boundary_contracts_and_configuration_validation.md +++ b/docs/ADRs/009_boundary_contracts_and_configuration_validation.md @@ -2,7 +2,7 @@ **Status:** Accepted **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase --- diff --git a/docs/ADRs/010_technical_risk_register.md b/docs/ADRs/010_technical_risk_register.md index 56d7bdf..993f9f3 100644 --- a/docs/ADRs/010_technical_risk_register.md +++ b/docs/ADRs/010_technical_risk_register.md @@ -2,7 +2,7 @@ **Status:** Accepted **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase --- diff --git a/docs/ADRs/011_replace_runtime_mapper_with_precomputed_lookup.md b/docs/ADRs/011_replace_runtime_mapper_with_precomputed_lookup.md index a19dc01..ed789c5 100644 --- a/docs/ADRs/011_replace_runtime_mapper_with_precomputed_lookup.md +++ b/docs/ADRs/011_replace_runtime_mapper_with_precomputed_lookup.md @@ -1,8 +1,10 @@ # ADR-011: Replace Runtime Spatial Mapper with Precomputed Lookup Table + + **Status:** Accepted **Date:** 2026-06-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase **Consulted:** FAO-FSFC (via Pre-release Note 02 and Release Note 02 confirmation) **Informed:** All contributors, UN FAO operational team diff --git a/docs/ADRs/012_revised_ontology.md b/docs/ADRs/012_revised_ontology.md index 433e70a..846292f 100644 --- a/docs/ADRs/012_revised_ontology.md +++ b/docs/ADRs/012_revised_ontology.md @@ -2,7 +2,7 @@ **Status:** Accepted **Date:** 2026-06-27 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase **Supersedes:** [ADR-001](001_ontology_of_views_postprocessing.md) --- @@ -14,7 +14,7 @@ a **runtime spatial-mapping engine** and **bundled shapefiles** as the authorita stable core. That architecture no longer exists: - [ADR-011](011_replace_runtime_mapper_with_precomputed_lookup.md) replaced the runtime - geopandas mapper with a **precomputed GAUL lookup table** (`GaulLookupEnricher`). + geopandas mapper with a **precomputed GAUL lookup table**. - The mapper, shapefiles, and caching machinery were deleted (C-39 / PR #42). - Input-integrity invariants and structured delivery were added (epic #51). @@ -58,9 +58,9 @@ described the cut-a-repo case; it now says which is which. | Category | Purpose | Authority | Stability | |----------|---------|-----------|-----------| | **Delivery Invariants** | Representation-free rules over primitives that a delivery must satisfy: coverage, no-collapse, gid parity, observed-range, provenance. Live in `delivery/` — **nothing there imports pandas or views_frames**. *Forecast identity was one of these until 2026-07-31 — see the amendment below.* | Authoritative — they define what a valid delivery is | Stable — changes are governance decisions | -| **Representation Seam** | `contract/frame_extraction.py` — turns a `views_frames` frame into the primitives the invariants consume. **One seam.** Its pandas sibling `unfao/extraction.py` was deleted in #151 once the pandas delivery was retired; the two ran as deliberate WET siblings through the migration. | Derived — isolates the representation so invariants stay representation-free | Evolving | +| **Representation Seam** | `contract/frame_extraction.py` — turns a `views_frames` frame into the primitives the invariants consume. **One seam.** Its pandas sibling `unfao/extraction.py` was deleted in #151 once the pandas delivery was retired; the two ran as deliberate WET siblings through the migration. | Derived — isolates the representation so invariants stay representation-free | Evolving | | **Wire Mechanism** | `contract/wire/` — the ADR-013 contract: header, shard, sidecar, run manifest, sink, source selection. Partner-neutral: it takes its consumer name and collapse floor as **arguments** (#153). | Authoritative — the contract with the consumer | Stable — changes are contract amendments | -| **Enrichment Asset** | The precomputed GAUL lookup (`data/gaul_lookup.parquet`), its identity in `contract/gaul_lookup.py`, its schema in `contract/gaul_schema.py`, and the keyed gather that joins it (`contract/enrichment.py`, the build/verification path — numpy/pyarrow since #89; see register **C-75** on whether that class should survive at all). | Authoritative for geographic metadata | Stable — rebuilt only when the producer releases new GAUL data | +| **Enrichment Asset** | The precomputed GAUL lookup (`data/gaul_lookup.parquet`), its identity in `contract/gaul_lookup.py`, its schema in `contract/gaul_schema.py`, and the keyed gathers that join it — `contract/historical.py` for the actuals artifact and `contract/wire/sidecar.py` for the §5 sidecar. *(A second gather lived in `contract/enrichment.py` with no production caller; retired in #90, register **C-75**.)* | Authoritative for geographic metadata | Stable — rebuilt only when the producer releases new GAUL data | | **Artifact Builders** | `contract/historical.py` — turns a frame plus the lookup into the partner-facing artifact. | Derived | Evolving | | **External Facts** | Facts read from systems this repo does not own: the producer's (`contract/source_metadata.py` — `last_valid_month_id`, D-07) and the store's (`contract/store_metadata.py`). | Authoritative (the owning system is the source of truth) | Evolving | | **Launch Declarations** | `contract/launch_config.py` — the delivery mode the launcher must declare. Omitting a key is **refused by name**, never inferred (ADR-003, register C-63). | Authoritative | Stable | @@ -70,10 +70,10 @@ described the cut-a-repo case; it now says which is which. **Two claims this ADR made until 2026-08-01, both now corrected rather than quietly dropped** (register C-67). It called the manager *"the **thin** `UNFAOPostProcessorManager`"* when it was -636 lines holding two of everything, and it called `unfao/extraction.py` *"the **single** +636 lines holding two of everything, and it called `unfao/extraction.py` *"the **single** pandas-aware module"* when pandas lived in three. Both drifted the same way: the ADR described the intended end state of a migration that then stopped one step short. Both are now true — -pandas has **zero runtime importers** (#89 made `contract/enrichment.py`'s a type-only import under `if TYPE_CHECKING` — pandas is in that class's interface, not its implementation) — and the load-bearing ones are +pandas is **absent from the package entirely** (#89 reduced the last one to a type-only import; #90 retired the module that held it — register C-75) — and the load-bearing ones are **mechanically checked** by `tests/test_doc_accuracy.py`, so the next drift fails CI instead of waiting for an audit. @@ -129,7 +129,7 @@ Seam" wording will need a light touch (the seam stays; its internals change). ## Amendment 2026-07-31 — forecast identity re-homed to the wire layer (#150, epic #148) -**`views_postprocessing/delivery/identity.py` is retired.** The "Delivery Invariants" row +**`views_postprocessing/delivery/identity.py` is retired.** The "Delivery Invariants" row above listed *forecast identity* among the authoritative rules; that module no longer exists, and this amendment records where the rule went so a reader of the row is not looking for deleted code. diff --git a/docs/ADRs/013_sampled_forecast_wire_contract.md b/docs/ADRs/013_sampled_forecast_wire_contract.md index 3f8e57c..35099b5 100644 --- a/docs/ADRs/013_sampled_forecast_wire_contract.md +++ b/docs/ADRs/013_sampled_forecast_wire_contract.md @@ -2,7 +2,7 @@ **Status:** Accepted **Date:** 2026-07-15 -**Deciders:** Project maintainer (PRIO MD&D Team) — explicit sign-off, views-models#149 +**Decider:** Simon Polichinel von der Maase — explicit sign-off, views-models#149 **Supersedes:** the "platform ADR-046" that platform issues cited as the format authority — no such *format* document ever existed (a pipeline-core ADR-046 exists but covers storage infrastructure only; see Erratum E2 in the Post-adoption record) — and the v1 proposal comment on views-models#149 (2026-07-02). --- @@ -352,11 +352,53 @@ identifier arrays mean (`time` is the VIEWS month-id; `unit` is the `priogrid_id this platform has already paid once for leaving id vocabulary implicit (the gid/id epic: a past platform-wide cleanup needed just to disambiguate what its integer identifier columns meant). `provenance` is exactly the three keys shown (strings/bool). **Caveat:** -`pipeline_core_version` is self-reported and will be unreliable until pipeline-core's -release train (their #261) cuts real releases (status at adoption, 2026-07-15: none -yet) — consumers must not treat it as authoritative before then. The lift of this -caveat is tracked as a reminder in pipeline-core: their issue #279 (filed -2026-07-19) fires when the first real release ships. +`pipeline_core_version` is self-reported. It was declared unreliable at adoption +(2026-07-15) because pipeline-core's release train had cut no real releases yet, and +consumers were told not to treat it as authoritative. **That caveat is lifted in part — +see Erratum E3 below, which also explains why "in part" is the accurate word.** + +**§2.2a Amendment A2 — run maturity, drafted 2026-08-05, NOT YET IN FORCE.** + +Requested by views-postprocessing #133 on behalf of views-faoapi (its ADR-033, epic +#244) so that a consumer can tell a production run from a test one. This clause defines +the field and the cost of adopting it. **It is not adopted, and `contract_version` +remains 1.5.** Nothing in this section describes bytes that ship today. + +*The field.* A fourth key inside `provenance`: + +| key | type | meaning | +|---|---|---| +| `status` | string | the producing run's maturity in views-models ADR-017's vocabulary — `graduate`, `candidate`, and the rest. Never inferred; absent is not `graduate`. | + +*Why this is an amendment and not a free addition.* §2.1 clause 1 opens the header to +new **top-level** keys and closes the three sub-objects. `provenance` is one of them, so +a fourth key is a contract change by this document's own rule. It is purely additive, so +it is a **MINOR** bump — 1.5 → 1.6 — and a reader built for 1.5 still accepts a 1.6 +artifact. + +*What adoption actually costs, which is the reason it is deferred.* `contract_version` +is written **inside the header bytes**, and §10 pins those bytes. So a MINOR bump is not +a free-standing edit: it rebuilds the golden fixture, and §10 requires all three +implementing repositories to re-vendor the new bytes and re-pin their root hashes +together. The full coordination cost is paid for one optional field. + +*Who can stamp it, and it is not this repository.* Hop-B shard headers are the Hop-A +headers **re-embedded untouched** (`contract/wire/sink.py`). This repository forwards +provenance; it does not compose it, and it does not know a run's maturity. Synthesising +one here would be a producer guessing at a fact it was not told — precisely the +inference this platform's declare-don't-infer rule exists to forbid. **The stamp belongs +to the Hop-A producer (views-models),** and this repository's part is to forward it, +which requires no code change once the key is admitted. + +*Named trigger for adoption (ADR-014 §4).* Adopt A2 **when the next re-vendor of the +§10 fixture happens for another reason** — the pyarrow fix tracked as register C-72 / +issue #174 is the one currently expected, since it also changes delivered bytes and also +requires all three repos to move together. Riding along costs nothing extra; going alone +costs a three-repo coordination for a field the consumer has stated it only surfaces. +**Owner:** whoever executes that re-vendor. If views-faoapi ever promotes `status` from a +surfaced label to a serving gate, that supersedes this trigger and A2 should be adopted +on its own. + **§2.3 Governance hook.** Changing `sample_count` (wire thinning) or `dtype` on the **FAO delivery** changes the published HDI/MAP numbers, and is therefore a @@ -890,6 +932,53 @@ runs can therefore never touch the live bucket by accident. Dated events after adoption. Errata correct errors in this document; other entries record execution progress against it. +- **2026-08-10 — Erratum E3 (§2.2, `pipeline_core_version`; `contract_version` stays + 1.5):** the adoption-time caveat said this field would be unreliable until pipeline-core + cut a real release. **views-pipeline-core 3.0.0 shipped to PyPI on 2026-08-03**, so the + condition is met — but the honest lift is narrower than "the field is now reliable", and + splits by how the *producing* pipeline-core was installed. + + **Authoritative when the producer ran a released distribution.** A wheel's recorded + version is written by the release that built it, so it cannot disagree with the code + beside it. In force now, for 3.0.0 onward. + + **`"unknown"` when the producer ran an editable install** — and this half is **not yet in + any released version**, which is the part the originating request (#228) stated too + strongly. An editable install's recorded version is fixed at the moment `pip install -e` + last ran and never tracks the source afterwards; pipeline-core#403 makes that case report + `"unknown"` instead of a stale number. That fix merged **2026-08-04**, a day and a half + *after* 3.0.0 was uploaded (2026-08-03 02:06 UTC), so **no released version contains it.** + It becomes true of producers at pipeline-core's next release. + + Verified here rather than taken on trust, 2026-08-10: in this repository's development + environment `importlib.metadata` reports pipeline-core **2.3.0** while the source beside + it is **3.0.0** — a full major version stale, exactly the value that would have been + stamped into published provenance. With the fix present, `_pipeline_core_version()` + returns `"unknown"` instead. + + **What consumers must do, and it is the operational point.** Treat `"unknown"` as *"do + not infer the producing version"*, never as an error. The set of runs producing + `"unknown"` will **widen** at pipeline-core's next release, because every developer run + joins it. A consumer that starts rejecting `"unknown"` on the strength of this lift would + break exactly those runs. + + This repository does not produce the value — Hop-B re-embeds the Hop-A header untouched + (`contract/wire/sink.py`) — so nothing here changes. The field is declared by this + contract, which is why the lift is recorded here. Arises from #228; pipeline-core #279 + (closed), #403, and their register C-280. + +- **2026-08-05 — Amendment A2 drafted, not in force (§2.2a; `contract_version` stays + 1.5):** views-postprocessing #133 asked for three declared fields on the run manifest — + `maturity`, `source`, and a required schema version. Measured against what is actually + delivered and actually read, **two of the three already ship**, and the third is not + ours to stamp. `source` is `provenance.ensemble`, forwarded from the Hop-A header and + read by views-faoapi as exactly that. The schema version is `contract_version`, present + on the run manifest and in every shard header. Only maturity is missing, it belongs in + `provenance`, and `provenance` is a closed sub-object — hence A2 rather than a silent + addition. The request was written against the manifest; the consumer reads the shard + header. Recorded because an accepted ask nobody re-measured would have produced a + three-repo fixture re-vendor to add two fields that were already there. + - **2026-08-02 — Erratum E2 (§7d links; `contract_version` stays 1.5):** §7d's first link was **broken and had never resolved**. Issue #158 renamed the cross-repo contract from `PLATFORM-001` to *the Appwrite Seam Contract* and the rename was applied as a @@ -1120,7 +1209,7 @@ record execution progress against it. **4.73 GB peak, 108 shards, 5 min, zero store calls** (interlock held). **(B) the historical path is pandas-free** (#126): actuals fetched as a `views_frames.FeatureFrame` (the frame path's FIRST production consumer — - C-40's gate lifted), artifact built by `unfao/historical.py` via pyarrow — + C-40's gate lifted), artifact built by `contract/historical.py` via pyarrow — reader-level parity with a legacy characterization golden proven through faoapi's own reader semantics. Two ghosts found in the legacy artifact and deliberately exorcised (faoapi reader verified safe on both): junk `row`/`col` @@ -1142,8 +1231,8 @@ record execution progress against it. passed at 128 draws, sidecar + manifest staged, zero store calls. - **2026-07-20 — HOP-B SINK LEG SHIPPED (epic #105 complete; upload-disabled).** The contract's missing middle exists in fixture-proven code: - `unfao/wire/` (naming, header, shard, sidecar, run_manifest, source_selection, - sink) + `unfao/product.py` + `delivery/parity.py`, wired into the manager + `contract/wire/` (naming, header, shard, sidecar, run_manifest, source_selection, + sink) + each partner's `product.py` + `delivery/parity.py`, wired into the manager behind an explicit declared `wire_contract` launch key. Settled by shipping: **§4.2a's configuration home is `unfao/product.py`**; **§5.2's parity invariant is `delivery/parity.py`**; the §11.4 upload interlock is live in @@ -1177,7 +1266,7 @@ record execution progress against it. merged 2026-07-15 and run-0 uploaded 2026-07-27, so the sequencing constraint was satisfied. The legacy reader it protected has now been **deleted**: this repo reads forecasts only through the contract path (manifest selection, - `unfao/wire/source_selection.py`) and historical actuals only as a + `contract/wire/source_selection.py`) and historical actuals only as a `views_frames.FeatureFrame` (#126). **The retired guard, recorded here because the code that carried it is gone.** @@ -1204,7 +1293,7 @@ record execution progress against it. this ADR should know the surrounding code changed shape and the wire did not. **What changed.** The pandas delivery this contract replaced was retired (#149) along - with the config fork that silently selected it; `delivery/identity.py` was retired + with the config fork that silently selected it; `delivery/identity.py` was retired because the contract path enforces declared identity **per shard header** instead (#150, §4.2a); the duplicate representation seam collapsed (#151); the GAUL lookup became one artifact read once (#152); and the partner-neutral machinery — the whole diff --git a/docs/ADRs/014_claims_and_the_guards_that_carry_them.md b/docs/ADRs/014_claims_and_the_guards_that_carry_them.md index 547c9f3..afddd36 100644 --- a/docs/ADRs/014_claims_and_the_guards_that_carry_them.md +++ b/docs/ADRs/014_claims_and_the_guards_that_carry_them.md @@ -2,7 +2,7 @@ **Status:** Accepted **Date:** 2026-08-02 -**Deciders:** Project maintainers (PRIO MD&D Team) +**Decider:** Simon Polichinel von der Maase **Arises from:** epic [#181](https://github.com/views-platform/views-postprocessing/issues/181) **Related:** [ADR-003](003_authority_of_declarations_over_inference.md) (declarations over inference), [ADR-005](005_testing_as_mandatory_critical_infrastructure.md) (testing as diff --git a/docs/ADRs/015_the_pipeline_core_appwrite_import.md b/docs/ADRs/015_the_pipeline_core_appwrite_import.md new file mode 100644 index 0000000..9648272 --- /dev/null +++ b/docs/ADRs/015_the_pipeline_core_appwrite_import.md @@ -0,0 +1,172 @@ +# ADR-015: Why this repository imports another project's Appwrite client + +**Status:** Accepted +**Date:** 2026-08-04 +**Decider:** Simon Polichinel von der Maase +**Arises from:** issue [#146](https://github.com/views-platform/views-postprocessing/issues/146), +recorded at the request of the þing-02 ratification +**Related:** [ADR-002](002_topology_and_dependency_rules.md) (dependency direction), +[ADR-009](009_boundary_contracts_and_configuration_validation.md) (boundary contracts), +[ADR-014](014_claims_and_the_guards_that_carry_them.md) §4 (a deferral needs a trigger), +register **C-40** + +--- + +## Context + +Each partner manager in this repository begins with two imports it would rather not have: + +```python +from views_pipeline_core.modules.appwrite.file import AppwriteConfig +from views_pipeline_core.modules.datastore import DatastoreModule +``` + +They reach into another project's Appwrite client — its auth handling, its provisioning +behaviour, its error semantics — and run it under this repository's identity and this +repository's API key. When that client does something surprising, it surprises FAO's +delivery, and the fix is in a repository this team does not own. + +This is knowingly kept. It has been raised repeatedly: register **C-40** carries it as a +Tier-2 concern, a cross-repository assembly examined it in July 2026 and told the platform +not to export the surface, and issue #146 tracks the eventual removal. None of that has +changed the code, and it should not have — but the *reason* has been recorded only in a +GitHub issue and in a deliberation folder outside this repository. A closed issue is not +where architectural reasoning survives. That is what this ADR is for, and writing it was +the condition attached when the deferral was ratified. + +**The word "deferred" is doing real work here and is easy to misread.** It does not mean +*not done yet*. It means: examined, decided, and held — with a stated condition under which +the decision is revisited. The distinction matters because the two read identically in an +issue tracker and completely differently to someone deciding whether they may change it. + +--- + +## Decision + +### §1 The import stays, because there is nothing to unwind to + +The obvious alternative is to write our own thin Appwrite client and drop the dependency. +We do not, for a reason that is about the platform rather than about this repository: +**a third copy of that client is the disease, not the cure.** + +Two hand-written copies already exist across the platform, and the defect that prompted the +cross-repository examination was common to both — not a divergence between them, but the +same mistake made twice. Adding a third copy here would multiply the surface the platform +is trying to shrink, and would do it in the repository that talks to the partner. + +The upstream project declines to export a supported client surface, and that refusal is +deliberate: exporting the current one would bless a shape its own maintainers have recorded +as needing decomposition. So the honest position is that the seam to unwind *to* has not +been built yet, by anyone, and building it unilaterally here would be worse than waiting. + +### §2 What bounds the blast radius while it stays + +A dependency kept on purpose still has to be contained, and this one is: + +- **Only the partner managers import it.** `tests/test_doc_accuracy.py` pins the importer + set to an explicit allowlist, so a third importer fails CI. Every other module — + the whole of `contract/` and `delivery/` — is free of it, proven in a fresh interpreter + by `tests/test_clone_readiness.py`. +- **The store is behind a port.** `_ContractStorePort` wraps the client in four methods, so + the wire modules never see Appwrite types. That is the dependency-inversion half of + C-40's prescribed mitigation, and it is the half that landed. +- **The framework contract is asserted.** `tests/test_framework_contract.py` checks that the + hooks we override still match upstream, that the classes remain instantiable, and that + every inherited attribute resolves — so an upstream change is found here rather than on a + delivery run. + +None of that removes the coupling. It makes the coupling *one file per partner wide* and +*visible when it moves*, which is the most that can be true while §1 holds. + +### §3 The condition under which this is revisited + +Two independent things can move, and they were ratified as a pair: + +- **Demand** — a second incident whose root cause is auth or provisioning handling in + duplicated client code, whether from divergence between copies or from a defect common to + them. +- **Supply** — the upstream project carving the auth and configuration seam out of its + Appwrite module, so that something exists to depend on. + +**Either one alone means revisit. Both together mean do it.** The deferral is tied to that, +explicitly *not* to this team's convenience — which is the wording the assembly chose, and +it is worth keeping because "when we get round to it" is how a deferral becomes a decision +nobody made. + +### §4 Where supply actually stands, which is not where the record says + +The verdict was written when the upstream Appwrite module was a single 3,064-line file. As +of the version this repository now pins, that has **partially** moved: provisioning has been +relocated to its own module, transport (including the request timeout this repository +depended on) to another, and an audit package now exists alongside. The main file is 2,841 +lines. + +**So the supply half has started and has not fired.** What we import is still there, and +there is still no exported client to depend on instead. But "no movement" would be the wrong +thing to believe, and a reader checking this in six months should check the module list +rather than trust this paragraph. + +### §5 Scope: this is two files now, not one + +The deferral was written when one manager held the import. The CRAF'd partner package added +a second, and it is a near-verbatim copy of the first. So the adapter to unwind is two +adapters with identical contents, and any fix must be applied twice or neither. + +That is not an argument to unwind sooner — it is the same bounded surface duplicated, and +register **C-33** already carries the duplication with its own trigger. It is recorded here +so that whoever eventually does the work is not surprised by the second file. + +--- + +## Consequences + +**What this costs.** Upstream changes land in FAO's delivery without this repository owning +the fix; the managers cannot be instantiated without the framework; and the platform's most +central package remains a hard dependency of the thing that talks to the partner. + +**What it buys.** No third client copy. The whole of `contract/` and `delivery/` stays +framework-free and reusable, which is what let a second partner be added as a package rather +than a fork. + +**Where this will go wrong first.** Someone reading issue #146 will see "deferred", read it +as "not done yet", and either do it — writing the third copy §1 exists to prevent — or treat +the import as unexamined and add a third importer. The allowlist test catches the second. +Nothing catches the first except this document. + +--- + +## Alternatives considered + +**Write our own thin client now.** Rejected in §1: it is the third copy, and it puts the +duplication in the repository with the partner relationship. + +**Wait silently and leave the reasoning in the issue.** This is what was happening, and it is +what the ratification objected to. An issue is a work item; it is not read by someone +deciding whether a rule applies to them. + +**Bound it with a test and consider it closed.** The allowlist and the port genuinely reduce +the risk, and it is tempting to call that the end. Rejected because containment is not +removal — C-40 stays open on exactly the residual this ADR describes, and closing it would +mean the next reader finds a bounded coupling with no record of why it is tolerated. + +--- + +## Appendix — the record this replaces + +The reasoning above was previously recorded only in issue +[#146](https://github.com/views-platform/views-postprocessing/issues/146) and in the +platform's deliberation folder (`views_platform/þingit/`), which is outside this repository +and not on any contributor's path. + +Two related facts, verified 2026-08-04 and recorded here because they are otherwise only in +that folder: + +- **The sibling obligation from the same verdict is discharged.** The assembly also required + this repository's legacy delivery path to be guarded or retired before 2026-11-30. It was + retired in #149 — the manager carries no such path, and register C-63 is resolved. The + deadline is moot here, and nobody had recorded that it was met. +- **`C-221` is the upstream project's register entry, not an issue number.** Issue #221 in + that repository is unrelated and closed. The entry is Tier 4 and describes the + decomposition need in §3's supply component. Cross-repository identifiers in this + repository are namespaced for exactly this reason + (`tests/test_register_integrity.py` enforces it), and the verdict's own shorthand is not. diff --git a/docs/ADRs/016_ci_read_access_to_private_siblings.md b/docs/ADRs/016_ci_read_access_to_private_siblings.md new file mode 100644 index 0000000..bb0cba9 --- /dev/null +++ b/docs/ADRs/016_ci_read_access_to_private_siblings.md @@ -0,0 +1,401 @@ +# ADR-016: Which sibling repositories CI downloads + +**Status:** Accepted +**Date:** 2026-08-10 +**Decider:** Simon Polichinel von der Maase +**Scope:** repositories CI **can** download — that is, public ones. What to do when a +repository **cannot** be downloaded is a different decision, taken in +[ADR-017](017_facts_across_a_private_boundary.md) and accepted alongside this one. +**Related:** [ADR-014](014_claims_and_the_guards_that_carry_them.md) §2 (a guard nobody has +watched fail is decoration) and §4 (a deferral names a trigger and an owner) + +--- + +## The decision in three sentences + +A few of this repository's tests read **other** repositories to check that things we say +about them are still true. Those tests only work when the other repository is on disk, so +they ran on a developer's laptop and skipped in CI. **CI now downloads the repositories +those tests need**, and a test fails if CI stops downloading one that the code says it +should. + +*Throughout, a "sibling" is another repository in the views-platform organisation that sits +beside this one in a developer's folder — never a dependency we install.* + +## What this document does not cover + +Downloading a repository requires being able to read it, so **everything here applies only +to repositories that are public.** A private one cannot be downloaded by our CI at all, +and no amount of workflow configuration changes that. + +That is not a gap in this decision; it is a different problem with a different answer, and +that answer is [ADR-017](017_facts_across_a_private_boundary.md). This document stops at +the boundary and says so, rather than implying a coverage it does not have. + +--- + +## Context + +### §1 What these tests actually do + +Most tests here check our own code. A few check something different: whether a **claim +this repository makes about a different repository** is still true. + +The clearest example. Each partner's `appwrite_env.py` records which edition of the +shared configuration registry it was verified against, by naming a commit in another +repository. Tests open that repository at that commit and ask whether anything this +package reads has moved since — a rename, a reclassification, a rotated value. *(Until +2026-08-11 they asked a cruder question — whether the edition **label** still matched — +which fired on every upstream edit. A second attempt also asked whether any coordinate had +**arrived**, which fired on other repositories' rows and was deleted on 2026-08-12. See +§7b's erratum.)* + +This is not hypothetical housekeeping. On 2026-08-05 that test failed — the registry had +moved to a new edition while nobody here was looking. Two days earlier, the same family of +checks fired **twice in one day**. + +### §2 Why they were not running + +A test like that needs the other repository present. On a laptop all the platform +repositories sit in one folder, so it runs. A GitHub Actions job gets **one** repository, +so it skipped. + +The result: **CI verified less than a laptop did, exactly on the checks that span two +repositories** — the ones no single repository can replace. + +### §3 Why nobody noticed for so long + +Which repositories CI could download was recorded in a **comment** in the workflow file. +That comment said `views-appwrite` was private, so its seven checks were assumed to need +an access credential nobody had issued. + +It had gone public on **2026-08-08**. The comment had not changed with it. Seven checks +stayed switched off for no reason at all, and the first draft of this very document +proposed issuing a credential to reach them — written one day after the thing that +justified it stopped being true. + +**That is the whole lesson.** A fact about another repository, written in prose, with +nothing able to check it, will eventually be wrong and nobody will find out. + +--- + +## Decision + +### §4 The list of siblings is code, not a comment + +`tests/conftest.py` holds one entry per sibling repository, and each entry says: + +- **`env`** — the environment variable that overrides where to find it; +- **`ci_checkout`** — whether CI downloads it; +- **`note`** — why not, when the answer is no. + +That is all. It replaces a comment with something a test can read. + +**Where that code lives, and why the question is worth asking.** The declaration sits in +`tests/conftest.py`, because that is where this repository already keeps facts several +test modules share. It is not obviously the right home: `conftest.py` is pytest's fixture +file, and none of these declarations is a fixture — they are statements about the platform +that tests happen to read. + +The cost is already visible. `scripts/build_gaul_lookup.py` needs the same +sibling-location fact and cannot import it, because a script importing from `tests/` is +the dependency direction backwards. So it declares the variable name a second time. That +duplication is defensible on its own merits — the two contracts genuinely differ — but it +was not a free choice, and a reader should know that a structural constraint and a design +decision happened to agree. + +**Named trigger for moving it (ADR-014 §4):** a second non-test consumer needing one of +these declarations, or a fifth declaration arriving in that file. The shape it would move +to is a small module owning the platform declarations, importable by tests and scripts +alike. Not committed to here, because the second incident is what tells you whether that +shape is right. Tracked as register **C-88**. + +Who is who, once and in one table: + +| repository | public? | fetched by CI? | why not, if not | +|---|---|---|---| +| **views-appwrite** | yes | **yes** | — carries the registry drift checks | +| **views-crafdapi** | yes | **no**, since 2026-08-12 | it was fetched for one test; that check read the consumer's source, broke twice in a day when both consumers improved theirs, and was deleted rather than repaired (ADR-017 §7 and its §5 erratum) | +| **views-datafactory** | yes | no | its checks need raw data that is not in its git repository; fetching it turns an honest skip into a crash | +| **views-faoapi** | **no** | no | private — no amount of workflow configuration reaches it. That case is [ADR-017](017_facts_across_a_private_boundary.md) | + +**What of this table is enforced, and what is not.** `ci_checkout` is: the workflow and +`SIBLINGS` are compared by `tests/test_ci_sibling_coverage.py`, and disagreement fails. +The **`public?` column is not, and cannot be** — which is why `public` was deleted from the +`Sibling` record on 2026-08-10, after a review found nothing could verify it. It is here +because a reader needs it to follow the argument. Nothing reads this table; if it drifts +from `SIBLINGS`, only a human will notice. + +### §5 CI downloads exactly what that list says, and a test enforces it + +The workflow downloads every sibling marked `ci_checkout=True`. +`tests/test_ci_sibling_coverage.py` fails if the workflow and the list disagree — in +either direction: a repository the list expects and CI does not fetch, or one CI fetches +that the list never mentions. + +Six rules, and each exists because of something that has actually gone wrong: + +| | rule | the incident behind it | +|---|---|---| +| G1 | a listed repository is downloaded, and the tests are pointed at it | the tests looked in the wrong place and skipped silently | +| G2 | anything CI downloads appears in the list | the list is the thing people read; CI is not | +| G4 | a repository we skip says why, and **every** note names a record | silent non-coverage reads as "nothing to see here" — and a note nothing points at is the prose this file replaces, whether it explains an exclusion or a temporary inclusion | +| G5 | a download step may not be marked "ignore failures" | one setting and a failed download stops failing the build | +| G6 | downloads land in `_siblings/` | a sibling put elsewhere made the linter report 745 errors in someone else's code | +| G7 | download the sibling's `main` branch | a sibling's default branch is not ours to rely on — views-appwrite's was `development` as of 2026-08-10, and is theirs to change again | + +Each rule is a plain function, so each is also run against a deliberately broken example +to prove it objects. A rule only ever tried against a correct file is a rule nobody has +watched fail. + +**One of these downloads was temporary, and it is worth knowing what became of it.** +`views-crafdapi` was fetched for exactly one test — a check that read the consumer's own +source. [ADR-017](017_facts_across_a_private_boundary.md) §7 said this repository was never +entitled to depend on another's file layout, and on 11 and 12 August 2026 both consumers +proved it by refactoring a literal into a named constant and reddening this repository +twice in a day. The check was deleted rather than repaired, and the download went with it. +What that costs is register C-92: nothing here verifies that a consumer's query uses the +name it declares. + +### §6 A repository CI expects but cannot find turns the build red + +Skipping is right on a laptop, where a missing sibling is normal. It is wrong in CI once +we have said the repository should be there — a silent skip puts us back in §2 while +looking fixed. So CI names the repositories explicitly, and a missing one is a failure. + +That guard existed already +(`tests/test_env_declaration.py::test_no_sibling_override_points_at_a_missing_path`) and +had a hole, closed here: it treated a variable set to an **empty string** as unset, which +is exactly what a mis-typed CI setting produces. So the likeliest misconfiguration was the +one case it could not see. + +### §7 A broken sibling can block merging here, and that is accepted + +This is the real cost, and it should not be buried. CI now depends on two other +repositories. If one of them changes in a way that fails a check — say the configuration +registry moves again — **this repository's builds go red and merges are blocked until +someone updates the pin.** Since merging to `main` here *is* the release to FAO, that +matters. + +It is accepted for two reasons: + +1. The problem being fixed was that these checks were **invisible**. A check that reports + but cannot block is invisible again, just more politely. +2. A red build in that situation is *correct*. It says "re-pin before you ship", and the + fix is a small edit rather than an investigation. + +**A third reason was offered and withdrawn, because it was not true.** An earlier draft +said the maintainer administers this repository and can therefore merge over a failing +check when something is genuinely urgent. Two external reviewers challenged it, and it does +not survive measurement: the `protect_main` ruleset lists **zero bypass actors**, and a +ruleset applies to everyone except the actors named there — so administrator status confers +no exemption. There is no classic branch protection either, hence no `enforce_admins` route. + +So **there is currently no escape hatch**, and the coupling here is accepted without one. +That is defensible — the two reasons above stand on their own — but it should be a chosen +position rather than a surprise on the day it matters. Adding a bypass actor is a console +change and would restore the third reason; it has not been made. And an override of that +kind would be one person's judgement, available only while that person is — the same +habit-dependence this document criticises in its own alternatives. + +### §7a How often this actually bites + +*"A small edit"* reads differently at once a quarter than at once a day, so the rate +belongs here rather than in a reader's imagination. + +views-appwrite reports its registry moved through **five editions in four days** — v1.4.0 +on 2026-08-02 through v1.4.4 on 2026-08-05. **Four of the five were observation-driven**, +recording what a console showed or correcting a key's scopes, and carried no obligation for +any consumer. Under this section each would have reddened this repository and blocked a +release until someone re-pinned. + +That is not an argument against the decision; it is the honest size of it. It also points +at a better shape, which that repository has volunteered to make usable: pin against the +contract **version** and treat observation-only bumps as non-blocking, rather than pinning +a commit and blocking on every edit — the upstream amendment log already marks which bumps +carry obligations. **Not adopted here**, because it needs the upstream side first and this +document should not decide another repository's format. + +### §7b Two kinds of coupling, and only one is a tripwire + +A reader could take G7 as *"always track the moving tip of `main`"*, which would sit oddly +beside the rest of the platform, where consumers pin the seam contract by tag and never by +branch. Both live here, and they answer different questions: + +- **Drift tripwires** read the sibling's `main` *because* movement is the signal. The + registry-edition check is one: if the registry moved and our pin did not, we want to know, + and a red build is the entire point. +- **Reachability checks** verify that a *pinned* commit is an ancestor of the sibling's + `main`. Here movement is noise; what matters is that what we pinned was ratified rather + than taken from someone's unmerged branch. + +G7 makes both read `main` rather than a default branch. It does not make either of them +track the tip for its own sake. + +**Erratum, 2026-08-11 — there is a third kind, and §7a's "not adopted here" is stale.** + +§7a recorded views-appwrite's suggestion — pin against the contract *version* and treat +observation-only bumps as non-blocking — and marked it *not adopted here, because it needs +the upstream side first*. That framing was wrong in a way worth correcting rather than +quietly editing: it treated the noise as something only upstream could fix. + +The registry-edition check was matching on `meta.version`, a label meaning *"anything at +all changed"*. What this repository actually depends on is a set of rows. Matching on the +rows instead removes the false alarms **without needing anything from upstream** — which +is ADR-014 §3 applied properly: when a guard fires on something legitimate, the first +question is whether the matching is wrong, not whether the scope is too wide. The matching +was wrong. + +So the two categories above become three: + +- **Drift tripwires** read the sibling's `main` because movement is the signal. +- **Reachability checks** read a pinned commit; movement is noise. +- **Differential tripwires** read **both** — a pinned edition as the baseline, the + sibling's `main` as the comparison — and fire when something *we declare* differs between + them. This is the shape the registry check now has. It catches a rotation, which is + invisible to the other two, and it is silent through prose edits and version bumps. + + *(Erratum, 2026-08-12. This paragraph read "…or when a coordinate **arrives** in a table + we read" and called that the check's current shape. That half was deleted the same day: it + compared every row of every table this package depends on, so another repository's key + reddened this one. The cost of removing it — a coordinate issued **for** this package now + goes unnoticed until a human reads the registry — is carried in register C-90.)* + +*(Corrected twice on 2026-08-11, and the second time is the instructive one. The first +implementation compared against the sibling's **working tree** — so a developer whose clone +sat on a feature branch graded this repository against unreviewed content, which is #196's +shape. The first correction moved **one** check onto `origin/main` and this paragraph then +claimed all of them did; three others were still reading the working tree. All four read +`origin/main` now. A sentence written to describe a fix, one fix ahead of the code, is the +same defect §3 diagnoses.)* + +The third kind exists because the no-copy rule forbids writing expected coordinate values +into this repository. A pinned edition is the only lawful place to keep a baseline for +comparing them. + +views-appwrite#76 has since been done. It shipped on 2026-08-11 as registry version 1.6.0, +which adds a table naming each edition and saying whether that edition obliges the +repositories that read the registry. This repository does not read it yet, and does not have +to: the third kind of check above already tells us whether anything we depend on moved. What +the new table would add is upstream's own answer to the same question, which is cheaper and +does not require a pinned baseline at all. That is worth adopting, and it is not urgent. + +It is worth saying how we learned it had shipped. Nobody told us. The check that requires +every table upstream to be classified here went red the first time it met the new one, which +is what that check is for. + +--- + +This section overrides an earlier internal recommendation not to couple per-pull-request CI +to another repository at all. That recommendation's stated objection was coupling to another +repository's *default branch* — which G7 removes, by naming `main` explicitly instead of +accepting whatever default the other repository happens to be set to. The coupling that +remains is real, and is the trade described above. + +### §8 One repository stays out, and why that is not this document's problem + +`views-faoapi` is private. Our CI cannot download it, so the one check that reads it does +not run here. + +**No credential is issued to work around that**, and that is a decision rather than an +omission. Where that question *is* decided is +[ADR-017](017_facts_across_a_private_boundary.md): the fact gets declared somewhere public +that both sides can read, so that neither needs access to the other. + +**That decision is accepted but not yet implemented** — it depends on a declaration being +added to the platform registry, which is another repository's work. Until then the affected +check runs on a maintainer's machine and not in CI, and the sibling's `note` says so. + +Until that declaration exists, the affected check runs on a maintainer's machine and not +in CI. That is stated in the sibling's `note`, which rule G4 requires and which must name +the record that owns it. + +**Update 2026-08-11.** The declaration now exists (views-appwrite#75), views-faoapi +checks itself against it (their #379), and this repository's check reads the registry +rather than that repository's source. So the FAO half is no longer laptop-only, and no +credential was issued. CRAF'd's consumer-side check landed the next day (views-crafdapi#53, closed 2026-08-12). + +**Amended 2026-08-12.** Both source-reads are now gone, and the crafd fetch with them — +but not because those gates opened. The reads broke twice in twenty-four hours, each time +because a consumer improved its own code, so the mechanism was abandoned rather than +sequenced out. ADR-017 §5 carries the erratum; the residual is C-92. + +--- + +## Consequences + +**What this buys.** Seven cross-repository checks move from *"run when someone happens to +run them"* to *"run on every change"*. Two of them fired in earnest the week this was +written. Separately, the scan that refuses configuration **values** in this public +repository's documentation now runs on every pull request rather than only on a +maintainer's machine — worth having, since the README carried four such values once. + +**What it costs.** §7: another repository can block merging here. And the list of siblings +is now something a contributor must keep in step with the workflow, enforced by tests they +may not have read. + +**Where this will go wrong first.** Someone debugging a red build marks a download step +"ignore failures", or moves it out of `_siblings/`. Both are rules for that reason. + +--- + +## What was tried and removed + +An earlier version of this design also recorded, for each sibling, whether it was +**public** and the date that was last checked — with a rule pairing visibility against +whether the download used a credential. + +A review found the pair circular. The `public` field was read by exactly one rule, and +that rule existed to protect the `public` field's verifiability. + +**This is why §5's table runs G1, G2, G4, G5, G6, G7 with two numbers missing.** The +removed rules were **G3** (visibility must match whether a credential was used) and **G8** +(the date must be a plausible one). The remaining rules keep their original names rather +than being renumbered, so that anything written about "G6" still means G6. Nothing else consulted +either. Both were deleted on 2026-08-10 and no behaviour changed. The date field was worse +than useless: nothing could confirm the check had happened, so it manufactured confidence +rather than recording a fact. + +Visibility now lives in a sibling's `note` — prose, where it belongs, because no test here +can verify it in any case. What still holds without the flag is simpler and needs no +field: a repository CI cannot read fails to download, and G5 keeps that failure loud. + +Recorded because the removed design is more tempting than it looks, and because this +document once argued for it. + +--- + +## Alternatives considered + +**Leave it and rely on running the suite by hand.** The status quo for weeks. It works +exactly as well as one person's habits, which is not a property a safety check should have. + +**Copy the facts here instead of reading them.** Rejected outright: copies of that +configuration registry were the platform's original failure, and the standing rule is that +it is referenced and never copied. A test reading a local copy compares a thing to itself. + +**Ask GitHub whether a repository is public, from a test.** Rejected. It would be the only +network call in the suite, would need a credential to answer for private repositories — +the very thing in question — and would be unreliable exactly when a green build matters. + +**Issue an access credential and download the private one too.** Rejected, and the +reasoning is [ADR-017](017_facts_across_a_private_boundary.md) §9 rather than anything here: +a credential is the wrong shape of answer to a standing category, and there is a route that +needs no credential at all. + +--- + +## Appendix — checking it yourself + +``` +pytest -q +``` + +with the sibling repositories present is the full suite. Then: + +``` +VIEWS_APPWRITE=/nonexistent pytest -q +``` + +is what CI would see without the download — and it must **fail**, not merely skip. That is +§6. diff --git a/docs/ADRs/017_facts_across_a_private_boundary.md b/docs/ADRs/017_facts_across_a_private_boundary.md new file mode 100644 index 0000000..2710c15 --- /dev/null +++ b/docs/ADRs/017_facts_across_a_private_boundary.md @@ -0,0 +1,437 @@ +# ADR-017: Facts shared with a repository we cannot read + +**Status:** Accepted +**Date:** 2026-08-10 +**Decider:** Simon Polichinel von der Maase +**Scope:** what this repository does when it depends on a fact held in a repository it +cannot download — today because that repository is private, and in principle for any +reason. The companion decision, [ADR-016](016_ci_read_access_to_private_siblings.md), +covers repositories we *can* download. +**Related:** [ADR-003](003_authority_of_declarations_over_inference.md) (declarations over +inference), [ADR-013](013_sampled_forecast_wire_contract.md) §4.1a (the field in question), +[ADR-014](014_claims_and_the_guards_that_carry_them.md) §1 and §4 + +--- + +## The decision in one sentence + +**A fact shared across a boundary we cannot see through is declared in a public place both +sides read, and neither side reads the other's source code.** + +--- + +## Context + +### §1 The failure this is about + +When this repository delivers forecasts, it uploads each file with a label — a store +document field called `name`. The consuming API filters every one of its queries on that +label. + +If our label and their filter ever stop matching: + +- the upload **succeeds**; +- the file is stored, and paid for; +- the consumer's endpoint returns **empty**; +- **nothing, anywhere, raises an error.** + +ADR-013 §4.1a states it plainly: a document uploaded under the wrong name is *"invisible to +the consumer, not merely degraded."* Every system involved reports success while the +partner receives nothing. + +This is the failure mode that most deserves an automatic check, precisely because no human +or machine downstream will notice it. + +### §2 Who owns the label, which is not what you would guess + +ADR-013 §4.1a also says: the name is **"Config-owned by views-faoapi; changing it is a +contract amendment, not a deploy detail."** + +So the consuming API owns the value. This repository holds a **copy** of it — the constant +`CONSUMER_DOCUMENT_NAME` in each partner's `product.py`, which is what we actually write +onto every upload. + +That direction matters for everything below. The question is not *"can we see into their +repository"*. The question is **"they own a fact, we keep a copy, and how do we know the +copy is still right?"** + +**One honest limit on that claim.** ADR-013 states the ownership only for the FAO API — its +table names *"the literal faoapi model name"*, and the CRAF'd partner arrived later and is +not mentioned there at all. So consumer-ownership is written down for one partner and +merely assumed for the other. §7 treats them the same, which is almost certainly right, but +"almost certainly" is not a citation. **Settling that is part of the registry work**: a +declaration naming both partners makes the ownership explicit for both, which is a second, +smaller reason to do it. + +### §3 How we check it today, and why that stops working + +The existing check opens the consumer's source, finds the name their API filters on, and +compares it with our copy. + +That works when the consumer's code is on the same machine — true on a maintainer's laptop, +where all the platform repositories sit in one folder. It is impossible in continuous +integration when the consumer is private, because a CI job cannot download a private +repository without a credential. + +So the check runs when someone happens to run it, and not on the change that would break +it. + +### §4 Why this is a standing category and not a one-off + +Two consuming APIs exist. `views-crafdapi` is public; `views-faoapi` is private. Across the +whole organisation, **15 of 33 repositories are private** — private is ordinary, not +exceptional. + +The maintainer's instruction is explicit: `views-faoapi` may or may not become public one +day, but **there will be one or more private APIs at any given time**. So this document +must decide for the category, not for today's instance. + +--- + +## Decision + +### §5 The rule + +> **A fact shared across a public/private boundary is declared in the public contract +> surface. Each side verifies itself against that declaration. Neither side reads the +> other's source code.** + +The same thing as a picture: + +``` + views-appwrite + the registry (public) + "un_fao" + + ▲ ▲ + │ │ + views-postprocessing views-faoapi + writes the name reads the name + + these two never read each other +``` + +**The arrow that is missing is the point.** Neither repository reads the other, so it does +not matter that one of them is private, and no credential is needed by anyone. Both arrows +point at a third place that is public to everybody. + +Who does what, once and in one table: + +| repository | its role | what it checks | needs access to | +|---|---|---|---| +| **views-appwrite** | holds the fact | nothing — it is the authority | — | +| **views-postprocessing** (here) | writes the name onto every upload | its own copy against the registry | the registry only | +| **views-faoapi** (private) | opens files with that name | its own served name against the registry | the registry only | + +Concretely, for the delivery label: + +1. The value is declared once, in the platform's public coordinate registry + (`views-appwrite`, `docs/ADRs/platform/coordinate_registry.toml`) — **subject to + views-appwrite accepting the registry as the home; the request is views-appwrite#75.** + A store-document name is a contract fact rather than a coordinate, so admitting it + widens that registry's charter, and that is not this repository's call to make. (That + seat has since reviewed this document and agreed to implement it. The wording stands + anyway: the decision should rest on its own reasoning, not on assent presumed in + advance.) +2. **This repository checks its copy against that declaration.** No access to the consumer + is required, so the check runs in CI on the change that could break it. +3. **The consuming API checks its own code against the same declaration.** It needs no + access to us either — **because the registry is public**, not because this repository + is. The consumer never reads us at all, which is the symmetry the rule is really about: + neither side reads the other; both read a public third place. + +### The order these land in is part of the decision, not an afterthought + +**None of those three is in place yet**, and the present tense above describes the decided +end state rather than today's behaviour. + +**The obvious sequence has a hole, and it is green.** If step 1 lands, then step 2 replaces +the source-reading check with a registry read, and step 3 has not happened yet, the state +is: the registry declares a string a human typed; we check our copy against that string and +pass; the consumer checks nothing; and the check that *did* consult the consumer's real +source has been deleted. The build is green and what it proves is that **two values this +platform authored agree with each other.** That is strictly weaker than today, and it is +precisely the objection ADR-016 raises against copies — during that window the registry is +not an authority, it is a third copy, and nothing closes the loop. + +So the sequence is constrained: **step 2 adds the registry check but does not remove the +source-reading one. The source-reading check is removed only when step 3 lands**, and until +then it keeps running wherever it can — in CI for the public partner, on a maintainer's +machine for the private one. Slightly redundant for a while, and redundancy is the correct +price for not having a window where the only thing verifying a delivery label is our own +typing. + +**Erratum, 2026-08-12. The sequence above was followed, and the check was still the wrong +mechanism. Both facts belong here, and an earlier draft of this erratum got the first one +backwards.** + +Both partners' step 3 landed. views-faoapi#379 closed on 11 August (commit `8615574`); +views-crafdapi#53 at 12:15 on the 12th (commit `0c493ae`). So the constraint stated above was **satisfied**, not +bypassed — and the first version of this erratum claimed the opposite, that "neither +partner's step 3 is what removed them". That was false when written, six hours after the +second one landed. It is corrected here rather than quietly, because an ADR that repeals +its own rule on a false claim about another repository is §3's failure inside §5's text. + +**What actually happened is more useful than what that draft said, and it is checked +rather than asserted.** In both repositories the commit that satisfied step 3 was *the +same commit* that broke our source-reading check — verified: `8615574` and `0c493ae` each +introduce the named constant and add that repository's registry-binding test: each consumer, while binding its served name to the registry, tidied +`APIPathManager("literal")` into `APIPathManager(CONSUMER_DOCUMENT_NAME)`. The improvement +and the breakage were one edit. Twice, a day apart. + +That is the argument for removing the check, and it is **§7's argument, not §5's**: we were +never entitled to depend on another repository's file layout, and a mechanism that a +consumer breaks by improving itself will keep breaking. The ordering rule above did its +job; the thing it was ordering was not worth having. + +So the ordering constraint is not repealed — it is **discharged**, and superseded for this +class by a plainer rule: **a check that reads another repository's source is not a check +this repository builds.** The residual window is now permanent rather than temporary, +registered as **C-92**, and what closes it is each consumer proving its query uses the name +it declares — asked for in views-faoapi#390 and views-crafdapi#55, because that is a fact +only the consumer can hold. + +**One assumption worth stating rather than relying on.** The registry is versioned, so the +two sides could in principle read it at different editions and both pass while disagreeing. +That is not a live risk here because the label is contract-immutable — changing it is an +amendment, which produces a new edition both sides re-pin to, and our pin's reachability is +already checked. Recorded because an unstated assumption carrying a silent-failure mode is +what §1 is about. *(Until 2026-08-12 the check also read the consumer's source and +skipped in CI; see the erratum above — that half is gone and its residual is C-92.)* The order the three land in, and what is blocked on what, is +Appendix B. Said here because a decision record that reads as a description of the code is +how this repository has repeatedly ended up believing work was done. + +**"But you rejected copying."** [ADR-016](016_ci_read_access_to_private_siblings.md) turns +down keeping a local copy of the shared registry, on the grounds that a test reading a copy +compares a thing to itself. This is not that, and the difference is the whole point. + +We keep a copy of the label because we have to — it is written onto every upload, so it +must exist in our code. What ADR-016 forbids is a copy that is *also the thing we check +against*. Here the copy is checked against an authority held elsewhere, which is what makes +the check mean something. A copy nobody verifies is the failure; a copy verified against a +declared authority is just a value with a source. + +### §6 Why the coordinate registry, rather than somewhere new + +Four reasons, and the fourth is the one that makes this cheap: + +- **It is the right kind of place.** The registry already holds cross-repository facts + about the delivery seam, already distinguishes secret entries from non-secret ones, and + already records which repository consumes what. +- **It is public**, so a private consumer can read it without anyone granting anything. +- **It has a change process** and an amendment log, so a change to the label is a recorded + contract amendment — which is what ADR-013 §4.1a already says it should be. +- **This repository's CI already downloads and reads that file on every run.** Adding this + check costs no new machinery, no credential, and no new dependency. + +A label is not a secret. Knowing that documents are named `un_fao` grants no access to +anything; access is controlled by keys held elsewhere. So this belongs with the non-secret +coordinates and must never be placed among the secret entries. + +### §7 The rule applies to both partners, not only the private one + +`views-crafdapi` is public, so our check *can* read its source, and does today. It will +move to the registry anyway. + +The reason is not tidiness. If only the private case moves, the platform ends up with two +mechanisms doing one job — one reading a declaration, one reading source code — and a +future contributor has to know which partner uses which. One mechanism, applied uniformly, +is the point of having a rule at all. + +It also means this repository stops reading any consumer's source, which removes a +dependency on another repository's *file layout* — something we were never entitled to +depend on. + +**And it retires a download.** [ADR-016](016_ci_read_access_to_private_siblings.md) has CI +fetch `views-crafdapi`, and measurement shows that fetch serves **exactly one test** — this +one. Once the check reads the registry instead, that download buys nothing, and +`views-crafdapi` should be marked as not fetched, with a note saying why. Stated here +because a download that has quietly stopped earning its place is how the previous version +of all this went wrong, and because ADR-016 cannot know it: the decision that obsoletes it +is this one. + +### §8 What this does **not** verify, stated plainly + +This checks **our copy against the declaration**. It does **not** check the consumer's code +against the declaration. + +So if the registry says `un_fao`, and we write `un_fao`, and the consuming API quietly +starts filtering on something else — our check passes, and the delivery is still invisible. + +That half is genuinely the consumer's to verify, and cheap for it — it is the third item in §5's list, and needs no access to us because this repository is public. But it is +outside this repository's control, and there is a case where it may never exist: **a +private API operated by a third party**, who has no obligation to run any test of ours. + +**In that case the fact is not verifiable from here, and must be recorded as such** — an +accepted blind spot with a named owner, not a check that quietly covers nothing. Pretending +otherwise would be the exact failure ADR-014 §1 exists to prevent. + +### §9 What is deliberately not built + +- **No access credential** — no personal access token, no machine account, no GitHub App. + A credential solves one repository at a time and must be maintained forever, for a + category that grows. It is the wrong shape of answer to a standing problem, and it puts a + rotation obligation on one person. +- **No moving our checks into another repository's test suite.** It looks attractive — + the private side can read us for free — but it fails on its own extension case: a + third-party-operated API will not run our tests, and we would have no lever. A rule that + breaks on its second instance is not a rule. +- **No new file, service, or endpoint.** If a fact does not fit the existing registry, that + is information about the fact, not a reason to build a second surface. +- **No abstraction over "how to reach a private repository."** There is one such + repository, and the decision above is that we do not reach it. + +**Scope of the prohibition, narrowed after review.** Everything above is about **declarable +facts** — short values that can sit in a registry row. There is a second kind of shared +thing this rule does not reach: agreement about **behaviour**, such as whether two +independently written readers interpret a reserved entry the same way. You cannot put a +program's behaviour in a row, so §5's mechanism does not transfer, and a blanket "never a +credential" would leave that case with no answer at all. + +The rule does extend, but one level up: **declare the semantics rather than the code** — +state in the contract what a reader must do, and each repository tests its own reader +against that statement. Still no repository reads another's source, and still no +credential. That question is live elsewhere as views-models#327 / D-05, and this document is +an argument for settling it there rather than here. + +### §10 When to revisit + +- **A shared fact that genuinely cannot go in the registry** — for instance one that is + itself sensitive. Then this rule has met a case it does not cover, and the case is the + evidence needed to decide the next thing. +- **A third-party-operated private API becomes a real consumer**, making §8's blind spot + concrete rather than hypothetical. +- **The registry stops being public**, which would invalidate §6 entirely. + +**Owner:** the maintainer. None of these is a deadline; each is an event. + +--- + +## Consequences + +**What this buys.** The check that guards the invisible failure runs automatically, on the +change that could cause it, for every partner — with no credential, now or ever, however +many private APIs appear. Adding partner number three costs one registry row. + +**What it costs.** A change to a delivery label now requires editing a file in a third +repository. That is slower than editing a constant here, and deliberately so: ADR-013 +already calls this a contract amendment rather than a deploy detail, and this makes the +process match the words. + +**A dependency moves rather than disappearing.** We stop depending on the consumer's file +layout and start depending on the registry. That is a better dependency — declared, +versioned, public, with a change process — but it is not nothing, and if the registry +becomes unreliable this check inherits that. + +**Where this will go wrong first.** Someone changes the label in the consuming API and not +in the registry. Our check keeps passing, because our copy still matches the declaration, +and the delivery goes invisible exactly as before. **That is §8, and it is the whole +residual risk of this design.** It is why the consumer-side check is part of the rule and +not an optional extra. + +--- + +## Alternatives considered + +### Issue an access credential so our CI can read the private repository + +**Rejected.** It works, and it keeps both halves of the check in one place — genuinely its +strongest property. + +But the brief is a permanent category. A token is tied to a person, expires, and must be +re-scoped for each new private repository; a GitHub App is a substantially larger piece of +infrastructure to own. Either way the cost recurs per repository and per rotation, forever, +to verify a handful of short strings. And when it lapses, the checks skip and the build +stays green — the exact failure the whole of ADR-016 was written to end. + +### Move our check into the private repository's test suite + +**Rejected**, though it was the leading candidate for some time. + +The private repository can read this public one for free, so the check would cost no +credential and would fire in a real pipeline. Attractive, and correct today, when one +person maintains both. + +It fails on the case that defines the category. A private API operated by a **third party** +will not run a conformance test on our behalf, and we would have no way to require it. The +rule would then hold only for repositories we happen to operate — which is not a rule about +private APIs, it is a rule about our own repositories wearing a general-sounding name. + +### Accept the gap and check nothing + +**Rejected as the destination, though it is the honest description of where we are until +the registry entry lands.** The failure it leaves uncovered is the invisible one, and this +repository has already delivered an artifact nobody could find once. + +Worth stating because it is the fallback if the registry route stalls: the position would +then be *"this is not verified, here is the failure mode, here is who owns it"* — an +accepted blind spot, recorded. Never a check that appears to cover it and does not. + +--- + +## Appendix A — the facts behind §4 + +Measured 2026-08-10. Counts drift; the commands do not. + +``` +gh repo list views-platform --limit 60 --json name,visibility +``` +returned 33 repositories, 15 of them private. + +Consuming APIs: `views-crafdapi` public, `views-faoapi` private. `views-productionapi` and +`views-publicapi` are named in platform records but do not exist yet, so no default +visibility can be inferred for future APIs. + +## Appendix B — the work this decision requires, and in what order + +1. **views-appwrite** — declare the label for each partner in the coordinate registry. + Filed as views-appwrite#75. Nothing here can proceed before it. + + **Superseded 2026-08-11 — both rows now exist and both are in force.** This step used + to say the CRAF'd label was "not a value anyone has decided yet" and should follow its + data contract. views-appwrite declared both rows in v1.5.0, and the CRAF'd row states + the correction itself: *"Both sides already use this value in code today — this row + DECLARES a standing fact, it does not decide a new one."* It cites both sides' source. + + So step 1 is **done for both partners**, and step 2 covers both. What remains split is + step 3, which is per-partner and is what step 2's retirement half waits on. +2. **views-postprocessing** — switch the check to read the registry rather than the + consumer's source, for both partners. Ours, blocked on step 1. + + **Erratum, 2026-08-11 — this step used to say "in the same change, stop fetching + `views-crafdapi` in CI", and that contradicted §5.** §5 constrains the order: a + partner's source-reading check is removed only once *that partner's* consumer-side + check exists. The crafd source-read is the sole consumer of the crafd fetch, so + retiring the fetch retires the check — before views-crafdapi#53 has landed. The + instruction and the constraint could not both be followed. + + Corrected: **retirement is per partner, and follows that partner's step 3.** For FAO + that is now met (views-faoapi#379 merged 2026-08-11), so the FAO source-read goes. For + CRAF'd it was not yet, so the crafd source-read stayed — and with it the crafd fetch. + + *(Superseded 2026-08-12. views-crafdapi#53 closed at 12:15 that day, and the crafd + source-read was deleted anyway — not because the gate opened, but because the mechanism + was wrong. Both fetch and check are gone; `tests/conftest.py`'s note records it. See the + §5 erratum.)* +3. **views-faoapi** — verify its own served label against the declaration; §8's other half. + Filed as views-faoapi#379, and the consumer-side maintainer has accepted it. + +4. **views-crafdapi** — the same self-check for the other partner. Landed as + views-crafdapi#53 (closed 2026-08-12), with the *query*-side ask filed the same day as + views-crafdapi#55 alongside views-faoapi#390. + + The reasoning that made this leg necessary still stands: §7 retires the crafdapi fetch, + so without it the CRAF'd label would be checked against a registry row that nobody + checks against CRAF'd's actual code — no verification against reality at all, which is + worse than the status quo. §7 argues the rule must apply uniformly; uniform application + means uniformly filing the third leg. + +**Step 3 was a precondition for removing the old check**, though never for adding the new +one — see the sequencing note in §5. The order was safe to interrupt at any point: until +step 3 landed for a partner, that partner's source-reading check stayed, so no window +existed in which the label was verified only against our own typing. + +**That last sentence stopped being true on 2026-08-12, deliberately.** Both source reads +were deleted once the mechanism — not the ordering — was found wanting, so the window is +now permanent and carried as **C-92**. See the §5 erratum for why the constraint was +discharged rather than repealed. diff --git a/docs/ADRs/README.md b/docs/ADRs/README.md index 9e8ae78..b8fd52b 100644 --- a/docs/ADRs/README.md +++ b/docs/ADRs/README.md @@ -4,6 +4,12 @@ This repository uses Architectural Decision Records (ADRs) to govern structural, semantic, and operational behavior. +> **On the `Deciders` field (2026-08-10).** ADRs 000–015 originally recorded +> *"Project maintainers (PRIO MD&D Team)"*. That team no longer exists, and a decider +> field naming a body that cannot be asked anything is worse than useless to a future +> reader. All of them now name the sole decider, Simon Polichinel von der Maase. +> ADR-013's line keeps its original sign-off reference alongside the name. + ADRs are divided into two categories: 1. **Constitutional ADRs (000–009)** @@ -79,6 +85,35 @@ These ADRs form the architectural constitution of the repository. record in the same change. Arises from epic #181, which found seven places where this repo said one thing and did another. +- **ADR-015** — Why This Repository Imports Another Project's Appwrite Client + The partner managers import `views_pipeline_core.modules.{appwrite,datastore}` and run + another project's client under this repo's identity. Kept knowingly: a hand-written + client here would be the platform's *third* copy, and the upstream seam to depend on + instead does not exist yet. Records what bounds it (an importer allowlist, the + `_ContractStorePort` DIP port, a framework-contract test), and the two-part condition — + demand and supply — under which it is revisited. Arises from #146 and the þing-02 + ratification, which asked that the reasoning live here rather than in an issue. + +- **ADR-016** — Which Sibling Repositories CI Downloads + A handful of tests here verify claims this repo makes about *other* repos — chiefly that the + coordinate-registry edition we pinned is the one that exists. They need the sibling on disk, + so they skipped in CI and ran only on a laptop. Each sibling is now declared with its + visibility, the date that was checked, and whether CI fetches it; a test fails when the + workflow and the declaration disagree either way. Written after the workflow's comment + asserted a repository was private for two days after it went public — which is the argument + for a dated declaration rather than prose. The credential for the one genuinely private + sibling is deferred with a named trigger. + +- **ADR-017** — Facts Shared With a Repository We Cannot Read + The delivery label this repository writes is owned by the consuming API and mirrored here; + if the two drift the upload succeeds, the file is stored, and the consumer's endpoint is + empty with no error anywhere. Verifying the mirror currently means reading the consumer's + source, which is impossible in CI when that consumer is private — and private consumer APIs + are a standing category, not a one-off. Decides that such a fact is declared in the public + coordinate registry and each side verifies itself against it, so neither reads the other's + source. No credential, for any number of private APIs. States plainly the half it does not + cover: the consumer's own code against its own declaration. + ADRs numbered 010 and above define: - Domain-specific decisions diff --git a/docs/CICs/GaulLookupEnricher.md b/docs/CICs/GaulLookupEnricher.md deleted file mode 100644 index a49cfc5..0000000 --- a/docs/CICs/GaulLookupEnricher.md +++ /dev/null @@ -1,193 +0,0 @@ -# Class Intent Contract: GaulLookupEnricher - -**Status:** Draft -**Owner:** PRIO MD&D Team -**Last reviewed:** 2026-06-18 -**Related ADRs:** ADR-011 (replace runtime mapper with precomputed lookup) - ---- - -## 1. Purpose - -> Attach the 9 geographic metadata columns to a prediction frame by merging a -> precomputed GAUL lookup table on the PRIO-GRID cell id. - -It is the lookup-based replacement for `PriogridCountryMapper`'s runtime spatial -enrichment: the spatial computation has already happened upstream (the -views-datafactory area-majority join), so this class does only a table join. - ---- - -## 2. Non-Goals (Explicit Exclusions) - -- This class does **not** perform spatial computation — no geometry, no - shapefiles, no geopandas, no area-majority calculation. -- This class does **not** build the lookup table (that is - `scripts/build_gaul_lookup.py`, run offline). -- This class does **not** fill, impute, or invent metadata for unmatched cells. -- This class does **not** validate the result. Null/coverage enforcement lives on the - delivery path — `contract/historical.assert_metadata_complete` at artifact build and - `delivery/coverage.py` for the region contract. It is **not** the manager's - `_validate()`, which stopped null-gating in #149 and now asserts only that the read - resolved. (Corrected 2026-08-02; PR #200 retired the same claim in the manager's CIC - and this one was left standing.) -- This class does **not** read from the datafactory, viewser, or Appwrite. - ---- - -## 3. Responsibilities and Guarantees - -- Loads exactly one lookup Parquet at construction and verifies it carries the key - plus the 9 contract columns; missing columns raise at construction, and so does an - **empty** lookup — every cell would gather to null and the delivery would then - complain about missing metadata rather than about a missing lookup (S4 / #89). -- Returns the input frame augmented with exactly the 9 columns of - `gaul_schema.METADATA_COLS`: codes numeric, coordinates float, **names and iso as - `object`**. - *Changed in S4 (#89).* They were `category`, inherited from the pandas merge that - read the artifact's dictionary encoding. The gather that replaced it assigns plain - values. Measured on a 200-row output: category 1,795,191 bytes, object 60,061 — - a categorical carries the artifact's full 64,742-entry dictionary whatever the - output size. The builder still writes the artifact with dictionary-encoded names - (`# names/iso categorical (C-32 memory)`); that governs the file, not this output. -- A cell id present in the lookup is enriched with that cell's metadata. -- A cell id **absent** from the lookup yields **null** metadata for that row — - never a sentinel, never a fabricated value (fail-loud downstream). -- Row count, row order **and index** of the input are preserved. Pinned by - `tests/test_enrichment.py::TestFramePropertiesPreserved` across six shapes — - ordered, reversed, duplicated, single, empty, and a non-default index. - (An earlier draft of this line claimed *eight* shapes "including unknown gids", - counting a throwaway development script rather than the committed suite, and naming - a shape that class does not exercise. Unknown gids are covered, for null-value - correctness, by `TestFailLoud` — a different guarantee.) - On **empty** input this is now *more* true than before: the pandas merge replaced the - input's `RangeIndex` with an object-dtype `Index`, where the gather leaves it - untouched. The only behavioural difference found, and it is in the direction the - guarantee above already claimed. - ---- - -## 4. Inputs and Assumptions - -- A lookup Parquet exists at the configured path (default: the committed - `views_postprocessing/data/gaul_lookup.parquet`), indexed by `priogrid_gid`, - containing only fully-complete cells (no nulls, no `-1`, no empty strings). -- The input DataFrame has a column named by `pg_id_col` holding PRIO-GRID cell - ids; a missing `pg_id_col` raises `ValueError`. -- The lookup is the single source of geographic truth — the caller does not - expect this class to reconcile it against any other source. - ---- - -## 5. Outputs and Side Effects - -- Output: the input frame (or, with `only_metadata=True`, just `pg_id_col` + - `time_id_col`) with the 9 metadata columns attached by **keyed gather**. - *Changed in S4 (#89):* this was a pandas left-merge on the lookup's index. The - lookup is now read with pyarrow, sorted once, and addressed by `np.searchsorted`. - Attaching metadata to a frame is a keyed gather, not frame algebra, and it never - needed a merge — which is also what frees the builder to stop writing pandas index - metadata (S5 / #90). -- Public attribute: `lookup_version` — a short, stampable id read from the - lookup's **declared** `lookup_version` metadata key at construction - (`@<8-char source digest>`). Delegates to - `contract.gaul_lookup.version`. The manager stamps it on each delivery so a - delivery is traceable to the exact lookup build. - **It does not degrade.** An artifact carrying no declared key raises - `gaul_lookup.LookupVersionError` (logged at ERROR first, per ADR-008) rather - than returning a placeholder. Until S5 (#186) this returned the string - `"unknown"` whenever views-datafactory's ingestion-ledger shape moved under - it — silently, in the one field register C-15 exists to answer *after* a - suspect delivery. See register **C-60**. -- Side effects: logs the lookup size + version at construction (INFO); logs a - WARNING with the count and sample of unmatched cell ids when any occur; logs - ignored mapper-only kwargs at DEBUG. No file writes, no network. - ---- - -## 6. Failure Modes and Loudness - -- **Raises** at construction if the lookup file is missing or lacks a contract - column. -- **Raises** `ValueError` if `pg_id_col` is not in the input. -- **Does not raise** on unmatched cells — it surfaces them as nulls and logs a - WARNING naming the unknown cells, and separately counting rows that carried no - usable cell id at all. This is deliberate: enforcement is a single point on the - delivery path (`historical.assert_metadata_complete`), so a coverage hole fails - loudly there, not in two places. Passing a sentinel for unmatched cells would be a **bug** (it would - bypass that gate). Aligns with ADR-003 (fail loud on semantic ambiguity). - ---- - -## 7. Boundaries and Interactions - -- Allowed to depend on: numpy, pyarrow, `gaul_schema`, and a local Parquet file. - **pandas is interface-only** — callers hand this class DataFrames and get one back, - but nothing here constructs, reads or joins one, and its import is under - `if TYPE_CHECKING` (S4 / #89). `tests/test_doc_accuracy.py` asserts by AST that the - package has **zero** runtime pandas importers. -- Must **not** depend on: geopandas/shapely, the runtime mapper, the - datafactory, viewser, Appwrite, or any network resource. -- Treats the lookup table as an opaque, trusted artifact produced by the build - script; it does not re-validate the table's spatial correctness. - ---- - -## 8. Examples of Correct Usage - -```python -enricher = GaulLookupEnricher() -out = enricher.enrich_dataframe_with_pg_info( - df.reset_index(), pg_id_col="priogrid_gid", time_id_col="month_id", - only_metadata=True, -) -# out has the 9 metadata columns; unmatched cells are null. -``` - -Signature-compatible with the runtime mapper it replaced (same method name and key -kwargs), which is why the mapper-only kwargs are still accepted and ignored. -**The manager does not call this class** — it has no `enrich` reference at all, and -reads the lookup directly via `gaul_lookup.load()` (register C-66). This example is a -build/verification-path usage. Whether the class should survive that is register -**C-75**. - ---- - -## 9. Examples of Incorrect Usage - -- Filling unmatched cells with `-1`/`""`/`"unknown"` to "avoid validation - errors" — defeats the fail-loud contract and ships wrong data to FAO. -- Using it to enrich against a lookup built for a different region without - expecting nulls for out-of-region cells. -- Calling it expecting spatial recomputation when the lookup is stale — regenerate - the lookup with the build script instead. - ---- - -## 10. Test Alignment - -`tests/test_enrichment.py`: -- **Green:** the 9 columns present; values match the lookup; row count preserved; - codes numeric / coords float. -- **Beige:** lookup integrity (cell count, no nulls, no `-1`, dtypes); coordinate - formula (independent oracle). -- **Red:** unknown cell id and excluded ocean cells yield null; missing - `pg_id_col` raises. - ---- - -## 11. Evolution Notes - -- Stable: the 9-column contract and the fail-loud-on-unmatched behavior (changing - either is a coordinated change across this repo and views-faoapi). -- Expected to change: the lookup's cell-set/region and its source GAUL version, - via re-running the build script. Such regenerations must not change the schema. - ---- - -## End of Contract - -This document defines the **intended meaning** of `GaulLookupEnricher`. - -Changes to behavior that violate this intent are bugs. -Changes to intent must update this contract. diff --git a/docs/CICs/README.md b/docs/CICs/README.md index 177b1d6..4b2b88b 100644 --- a/docs/CICs/README.md +++ b/docs/CICs/README.md @@ -51,7 +51,6 @@ Contracts must be clear enough that: ## Active Contracts - `UNFAOPostProcessorManager.md` — Pipeline orchestration manager (read → transform → validate → save) -- `GaulLookupEnricher.md` — Precomputed GAUL lookup enrichment (ADR-011; replaced the runtime mapper) --- diff --git a/docs/CICs/UNFAOPostProcessorManager.md b/docs/CICs/UNFAOPostProcessorManager.md index 0178739..d8848a8 100644 --- a/docs/CICs/UNFAOPostProcessorManager.md +++ b/docs/CICs/UNFAOPostProcessorManager.md @@ -3,14 +3,14 @@ **Status:** Active **Owner:** PRIO MD&D Team -**Last reviewed:** 2026-06-02 +**Last reviewed:** 2026-08-05 **Related ADRs:** ADR-001, ADR-002, ADR-008, ADR-009 --- > **Corrected 2026-08-03 — this document named a collaborator the manager has never -> called.** Six statements described enrichment as delegated to `GaulLookupEnricher`, -> one naming the call `GaulLookupEnricher.enrich_dataframe_with_pg_info()`. The manager +> called.** Six statements described enrichment as delegated to `GaulLookupEnricher`, +> one naming the call `GaulLookupEnricher.enrich_dataframe_with_pg_info()`. The manager > contains **zero** references to it — `tests/test_gaul_lookup_access.py` actively > asserts its absence — and the sibling CIC has long said *"the manager does not call > this class."* Two contract documents asserted opposite things about the same call. diff --git a/docs/architecture/role_and_seams.md b/docs/architecture/role_and_seams.md index 1a4e99a..9af8da0 100644 --- a/docs/architecture/role_and_seams.md +++ b/docs/architecture/role_and_seams.md @@ -136,7 +136,7 @@ historical actuals arrive as a `FeatureFrame` via pipeline-core's `get_feature_f (#126 — this repo was its first production consumer), the forecast interior is `PredictionFrame`, and `contract/frames.py` is the **live constructor** every interior frame is built through — not the unused adapter it was when this was written, and it supports -`S > 1`. pandas survives in exactly one module, `contract/enrichment.py`, on the +`S > 1`. pandas is gone from the package entirely since #90 retired the last module holding it (register C-75); what remains is on the build/verification path. What **remains** of **register C-40** is narrower than this paragraph implies: the manager is @@ -181,7 +181,6 @@ views_postprocessing/ │ ├── historical.py the historical artifact, built pandas-free │ ├── gaul_lookup.py the GAUL asset: path, version, one read per delivery │ ├── gaul_schema.py the 9-column contract, declared as data -│ ├── enrichment.py GaulLookupEnricher (build/verification path; the last pandas) │ ├── source_metadata.py producer (datafactory) facts, e.g. last_valid_month_id │ ├── store_metadata.py prediction-store facts │ └── launch_config.py the delivery mode the launcher must declare @@ -204,7 +203,7 @@ views_postprocessing/ ## 7. Where to go next - **What was decided and why** → `docs/ADRs/` (esp. ADR-011 mapper→lookup; ADR-012 ontology). -- **Per-class contracts** → `docs/CICs/` (`UNFAOPostProcessorManager`, `GaulLookupEnricher`). +- **Per-class contracts** → `docs/CICs/` (`UNFAOPostProcessorManager`). - **Live risks / open constraints** → the technical risk register (C-40 the pandas gate, C-25/C-30/C-15 the delivery guards). - **The frame/draws future** → #45 (delivery-side draw carrier) and C-40. diff --git a/poetry.lock b/poetry.lock index 42a6e0f..bbc9180 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2641,7 +2641,7 @@ version = "6.0.3" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, @@ -3785,4 +3785,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.15" -content-hash = "a51d9be49b3be1984501750e17be52d4efb46d4cb72e2f1decaaf3bf999b9bf2" +content-hash = "302cbc3aa750f7dacffe848c6141e6ba542ce30f2f646d99a859409acfd23b49" diff --git a/pyproject.toml b/pyproject.toml index 19b8f99..d76c633 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,14 @@ pyarrow = ">=16.1.0,<17.0.0" pytest = ">=8.0,<10.0" # the suite runs on 9.x today; 8.x still works ruff = ">=0.6,<1.0" +# Declared for the same reason the pytest comment above exists. `tests/test_ci_sibling_ +# coverage.py` parses `.github/workflows/*.yml` to check that CI checks out exactly the +# siblings `tests/conftest.py::SIBLINGS` declares. pyyaml already arrives transitively +# (it is in the lock as a main-group dependency), and relying on that is precisely how +# this repository lost pytest. The alternative — `pytest.importorskip("yaml")` — would +# make the guard skip itself whenever the parser is absent, which is decoration. +pyyaml = ">=6.0,<7.0" + # Ruff's default rule selection is NOT stable across ruff versions, and inheriting a # moving default is the inference ADR-003 forbids applied to tooling. Measured # 2026-08-03: ruff 0.14.13 reported 0 findings on this tree; 0.16.1 reported **815**, diff --git a/reports/technical_risk_register.md b/reports/technical_risk_register.md index c2717ff..f2ebe2d 100644 --- a/reports/technical_risk_register.md +++ b/reports/technical_risk_register.md @@ -4,10 +4,10 @@ |-------------------|--------------------------------------| | Project | views-postprocessing | | Owner | Dylan Pinheiro / PRIO MD&D Team | -| Last Updated | 2026-08-03 | -| Total Concerns | 83 | -| Open Concerns | 17 | -| Resolved Concerns | 66 | +| Last Updated | 2026-08-12 | +| Total Concerns | 97 | +| Open Concerns | 24 | +| Resolved Concerns | 73 | --- @@ -49,7 +49,7 @@ covered a single open entry (see Historical clusters below). **Root cause:** the register, ADR prose, and issue bodies are hand-maintained mirrors of cross-repo state that moves under them. **Entries:** C-44, C-46, C-47, C-57, C-74 (a guard whose declared scan roots silently stopped existing — the cluster's disease inside the cluster's own prescription) — plus this register's own findings at review-rr 2026-07-31 (header miscount, two RESOLVED entries misfiled under Open, eight stale `unfao.py` line ranges after the manager grew 273→636 lines, two unnamespaced foreign-register IDs). Historical precedent: the entire C-48–C-55 ADR-013 audit series, and C-42/C-47. **Highest tier:** 3 -**Fix strategy:** this repo already solved this disease once — the ADR-013 audit series ended with **40 permanent guard tests** (`tests/test_falsify_adr013_*.py`), and the same pattern now guards the þing-01 invariants (`tests/test_env_declaration.py`, `tests/test_redaction_guard.py` — the latter briefly **only over the roots that still existed**, see C-74, resolved: a guard is only as good as the assertion that its inputs are real, and it now carries that assertion). There is **no equivalent for the register**. A small `tests/test_register_integrity.py` — header counts match section counts; no RESOLVED body under `## Open Concerns`; every `C-\d+`/`D-\d+` reference resolves or is namespaced to a foreign register — would make this class self-detecting. +**Fix strategy:** this repo already solved this disease once — the ADR-013 audit series ended with a **permanent guard suite** (`tests/test_falsify_adr013_*.py` — `pytest --collect-only -q tests/test_falsify_adr013*.py` for the count, which moves), and the same pattern now guards the þing-01 invariants (`tests/test_env_declaration.py`, `tests/test_redaction_guard.py` — the latter briefly **only over the roots that still existed**, see C-74, resolved: a guard is only as good as the assertion that its inputs are real, and it now carries that assertion). There is **no equivalent for the register**. A small `tests/test_register_integrity.py` — header counts match section counts; no RESOLVED body under `## Open Concerns`; every `C-\d+`/`D-\d+` reference resolves or is namespaced to a foreign register — would make this class self-detecting. **Resolution scope:** Full for the mechanical half. ### Cluster J: Delivery aftercare has no mechanism @@ -99,7 +99,7 @@ covered a single open entry (see Historical clusters below). | closed | by | proven by | |---|---|---| | **C-71** ADR-008 in the entry validator | S1 | a check parametrised over **both** validators, so the pair cannot drift again | -| **C-43, C-59, C-61** Cluster K's build-time guarantees | S2 | `tests/test_gaul_lookup_fidelity.py`, 26 tests | +| **C-43, C-59, C-61** Cluster K's build-time guarantees | S2 | `tests/test_gaul_lookup_fidelity.py` — one file discharged three entries | | **C-03** the `_validate` replica | S4 | 43 self-referential tests replaced by 14 against the real gate | | **C-60** the lookup's declared version | S5 | the artifact rebuilt, values byte-identical, stamp unchanged | | **C-57** coordinate-registry drift | S6 | four checks, each mutation-proven | @@ -114,7 +114,7 @@ Plus, outside the register: #158's rename finished and the **broken URL it creat **What the epic did NOT do, stated because a closeout that reports only successes is the defect this epic exists to fix:** 1. **Cluster M is untouched and correctly so.** Six entries, two of them Tier 2, all resolving on one upstream publish. No engineering here moves them. -2. **The CI question is decided in writing but not implemented.** Three gated cross-repo checks run nowhere automatic. C-46's residual carries the argued recommendation — *do not couple per-PR CI to another repo's default branch; if wanted, a weekly scheduled check that opens an issue on divergence* — with a named trigger. **It is a decision awaiting an owner, not a task awaiting effort.** +2. **The CI question is decided in writing but not implemented.** Three gated cross-repo checks run nowhere automatic. *(Largely implemented 2026-08-10 — ADR-016. Two of the three groups now run in CI; the datafactory group still cannot, because it needs raw data absent from that repository's git, which no CI change can supply.)* C-46's residual carries the argued recommendation — *do not couple per-PR CI to another repo's default branch; if wanted, a weekly scheduled check that opens an issue on divergence* — with a named trigger. **It is a decision awaiting an owner, not a task awaiting effort.** 3. **Withdrawal of a bad delivery is the chosen policy and is not built.** Supersession is in force because it is what the wire does. Deliberately not started: it needs an ADR-013 amendment plus views-faoapi work, and FAO's answer on audit requirements (Pre-Release Note 07, B.2) decides whether it is wanted at all. 4. **Two questions are with the UN FAO**, not with us — recipients and notification timing (B.1), withdrawal versus supersession (B.2). 5. **`test_datafactory_deploy_readiness`'s `xfail` tuning was left alone**, deliberately: S7 fixed how the checkout is found, not what the gate asserts. If it needs re-pinning now that datafactory has moved past `v1.4.0`, that is a separate judgement. @@ -161,6 +161,521 @@ that indexes only deleted code is noise. ## Open Concerns +### C-97: Coordinate values sit in docstrings and comments, where the scan deliberately does not look + +| Field | Value | +|-------|-------| +| ID | C-97 | +| Tier | 3 — a decided position, not an accident, and the decision is defensible. It is registered because the decision was taken before anyone counted, and because a docstring is printed by pytest in a way a code constant is not. | +| Source | `/code-review max` on #242, 2026-08-12; count reproduced here | +| Trigger | **Either.** (a) A check whose failing function contains one of these values starts firing in CI. (b) Someone proposes widening the no-copy scan to docstrings — at which point this entry is the measurement that says what that would cost. | +| Owner | This repository. | +| Location | Measured across `git ls-files '*.py'`, **25 standalone occurrences in 8 files** (2026-08-12, down from 33 in 9 when filed — #242 cleared the no-copy scan's own docstring and #248 cleared `tests/test_product.py`). Includes `views_postprocessing/{unfao,crafd}/managers/`, `contract/store_metadata.py`, `contract/wire/sink.py`. **The count moves whenever prose is edited; re-measure rather than cite it.** | + +The no-copy scan compares string **constants** and excludes docstrings outright. That exclusion is C-57's recorded lesson: an early draft fired on refusal labels and on docstrings naming which store a function serves, and *"a guard that fails on `def file_metadata(record)` gets deleted — after which the real rule is unguarded"* (ADR-014 §3). + +The position is still right. What was not known when it was taken is the count, and one consequence: + +**A value in a docstring is printed by pytest that a value in a constant is not.** When any test fails, `--tb=auto` prints the failing function's source. So a registry value in the docstring of a *test* reaches a world-readable CI log on that test's next failure — which is how #242 found that the no-copy scan's own docstring carried three. That one is fixed. The others sit in functions that fail less predictably. + +**Three of the values cannot be distinguished from ordinary code at all.** Measured: two are this repository's own package directory names, and one is the name of a function in `contract/store_metadata.py`. Exact-string matching cannot separate a copy from a coincidence for these, which is the same measurement #243 needs for the ban-set — recorded here once rather than twice. + +**What is deliberately not proposed:** scanning docstrings. It would fire on every sentence naming a store, which is the false-alarm class that gets guards deleted. The honest options are to leave it (current), to scan only *test* docstrings (where the traceback amplification is), or to stop writing values in prose going forward without rewriting history. None is urgent. + +Cross-refs: **C-57** (the exclusion and why), **C-89** (the traceback amplification, found here), **C-86**, issues #242, #243. + +--- + +### C-96: The registry table that says which live checks this package may build is classified as none of our business + +| Field | Value | +|-------|-------| +| ID | C-96 | +| Tier | 4 — no defect follows from it directly. It is here because it concealed an available mechanism for weeks, which is a cheap mistake to make again. | +| Source | `/expert-code-review` of the standing decisions, 2026-08-12 | +| Trigger | Anyone asks whether a live check against the production Appwrite project is permitted. | +| Owner | This repository. | +| Location | `tests/test_env_declaration.py:509` (`_TABLE_ROLE["test_environment"] = "IGNORED"`). | + +`[test_environment]` is classified `IGNORED` with the reason *"a fact about the platform, not about this package"*. It is in fact the clause that governs **what live checks this package is permitted to build**: + +> `status = "none"` — *"No non-production Appwrite project exists (þing-01 S23). Until the operator creates one: integration tests against the production project are FORBIDDEN by the seam contract; **read-only preflight validation is the only permitted live check**."* + +The second half is a **permission**, and this repository spent weeks believing the whole clause was a prohibition. Reclassify and read it, or record why a permission that changes what we may build is not a fact we depend on. + +Cross-refs: **C-94** (the mechanism this permission authorises), **C-95** (the mis-citation that compounded it), issue #249. + +--- + +### C-95: The integration-test prohibition is cited to the wrong verdict, in three places + +| Field | Value | +|-------|-------| +| ID | C-95 | +| Tier | 3 — a citation error, but one that closed off a design option. Namespacing across six repositories and two þings is a known hazard here; this is it landing. | +| Source | `/expert-code-review` of the standing decisions, 2026-08-12 | +| Trigger | Anyone reasons from the integration-test prohibition — for a preflight, a drill, or a new ADR. | +| Owner | This repository. | +| Location | `reports/technical_risk_register.md` (~:475, ~:1628, ~:1717); anywhere else citing þing-02 D2 for this. | + +This register cites **þing-02 D2** for the ruling that integration tests against the production Appwrite project are forbidden. þing-02 D2 is about identity and key separation. The ruling is **þing-01 D2** (`þingit/01_identity_secrets_config/orð_dómr.md:53-61`), and it differs from the paraphrase in two ways that matter: it is **conditional** (*"until the operator creates one"*), and it **grants** read-only preflight validation as the permitted live check. It also records that creating a test project is **assigned to the operator** and gates the provisioning-path drill — an open assignment, not a closed door. + +Cross-refs: **C-94**, **C-96**, þing-01 `orð_dómr.md` D2, issue #249. + +--- + +### C-94: No mechanism anywhere detects an invisible delivery at the time it happens + +| Field | Value | +|-------|-------| +| ID | C-94 | +| Tier | 2 — the failure mode is invisible by construction and lands on the live FAO path: upload succeeds, storage is billed, the consumer's endpoint returns empty, nothing raises anywhere. ADR-013 §4.1a's *"invisible to the consumer, not merely degraded."* | +| Source | `/expert-code-review` of the standing decisions, 2026-08-12 | +| Trigger | **Either.** (a) A delivery is reported empty by a consumer or by FAO. (b) `APPWRITE_READ_API_KEY` is provisioned for the launcher — at which point the deferral below has no remaining cost. | +| Owner | This repository, for the mechanism. The credential is the operator's. | +| Location | `views_postprocessing/contract/wire/sink.py` (the upload path, where nothing verifies); `views_postprocessing/delivery/`. | + +Every mechanism this platform has aimed at invisible delivery is a **CI-time proxy** for it: we check our label against the registry, the consumer checks their constant against the registry, and — until this week — we parsed their source. None of them observes the outcome. Grepping the sink and the delivery package finds no read-back, no findability check, no assertion that what was uploaded can be retrieved by the name the consumer will query. + +**The mechanism that would close it is known and is legal.** A producer-side **read-only findability preflight**: after upload, query the store read-only for a document whose `name` equals the declared `CONSUMER_DOCUMENT_NAME`; assert non-empty; log at ERROR and raise (ADR-008); remedy is the existing operator quarantine. It is authorised by the seam contract (see **C-96**), the `APPWRITE_READ_API_KEY` slot is already declared, and it is the only mechanism that survives a **third-party-operated private consumer**, because it asks nothing of them. + +**Why it is deferred, stated honestly.** It needs a read credential wired into the launcher — an operator action, not a code change — and it adds a live network call to the delivery path. Delivery works today. Building it now would be building the right thing at the wrong time. That is a deferral with a trigger and an owner (ADR-014 §4), not an omission. + +**What it would not cover, so nobody over-reads it later:** it proves the document is findable by that name in the store. It does not prove the consumer's code queries by that name. That last link is theirs, and issues asking each consumer to bind their *query* to their constant are filed under #248. + +Cross-refs: **C-92** (the check we deleted rather than replaced), **C-87** (the broader residual), **C-96** (the permission), ADR-013 §4.1a, ADR-017 §5/§8, issue #248. + +--- + +### C-93: A mutation proof written by whoever wrote the guard tests that author's imagination, not the guard + +| Field | Value | +|-------|-------| +| ID | C-93 | +| Tier | 3 — no defect of its own. It is the reason several of the defects below survived three reviews, and it changes what "mutation-proven" is allowed to mean in this codebase. | +| Source | `/code-review max` on PR #239 post-merge, 2026-08-11, corroborated by measurement | +| Trigger | The next time a guard is defended in a pull-request description as "mutation-proven" against a list of cases the same change authored. | +| Owner | Whoever writes the next guard; the standard belongs in ADR-014 §2. | +| Location | ADR-014 §2; `tests/test_env_declaration.py::test_no_coordinate_value_is_copied_into_this_repo`; every `_MUTANTS`-style proof in `tests/`. | + +ADR-014 §2 says a guard is mutation-proven or it is decoration. That is right, and it is not sufficient. **A mutation proof is only as good as the mutant list, and a mutant list written by the author of the guard covers the cases the author already had in mind — which are, by construction, the cases the guard already handles.** + +Measured, on the guard that protects a public repository from publishing a coordinate value. Its author (this assistant) proposed thirteen input forms, proved all thirteen caught, and wrote that result into C-57 and into a commit message. An independent review then proposed twenty-nine forms. **Fifteen missed.** The thirteen were not a sample of how people write markdown; they were a sample of what the author had thought of, and every one of them happened to share the property the guard depended on. + +This is not the same as ADR-014 §2 failing. The proof was real, it was executed, and every case in it genuinely passed. The gap is that "proven against N mutations" reads as a statement about the guard when it is a statement about N. + +**What would change.** Two candidates, and they are not exclusive: (a) for a guard whose failure is silent — a leak, an invisible delivery — the mutant list is written or extended by something other than the change that wrote the guard; (b) the mutant list is a *declared, named* artifact in the test file rather than a paragraph in a commit message, so the next contributor can see what was and was not tried. `tests/test_ci_sibling_coverage.py::_MUTANTS` is already (b) and is the pattern to copy. + +Cross-refs: **C-57** and **C-89** (the guard this was measured on), **C-90** (a proof that proved nothing at all), ADR-014 §2. + +--- + +### C-89: The guard against publishing a coordinate value publishes it — in one branch of three, proven on one side of two + +| Field | Value | +|-------|-------| +| ID | C-89 | +| Tier | 2 — a confidentiality exposure on a public repository with no signal that it happened. It fires on exactly the event the guard exists to catch, and CI logs are world-readable and are not retroactively redactable. | +| Source | `/code-review max` on PR #239 post-merge, 2026-08-11; every element verified in this repository | +| Trigger | A literal coordinate value is committed into a `.py` under `views_postprocessing/` — the violation the scan exists for — on any branch whose CI runs. | +| Owner | This repository. | +| Location | `tests/test_env_declaration.py` — `test_no_coordinate_value_is_copied_into_this_repo` (both reporting branches and the `secret` exemption in `scanned_sections`), `test_the_drift_check_would_catch_a_rotation_that_names_and_classes_cannot`. **Function names, not line numbers**: this entry has now cited stale ones twice, because each fix moved them. | + +The no-copy scan exists because README.md once carried four real coordinate values, two lines below the sentence promising they are never copied, in a public repository. It has three parts and they do not agree with each other about the one rule that matters. + +**The Python branch prints the value.** `copied.append(f"{source}:{node.lineno} = {node.value!r}")`. The markdown branch sixty-eight lines below was rewritten on 2026-08-11 to stop doing precisely this, and `_describe_changes` was added in the same change with a docstring whose entire subject is that a rotated coordinate must never reach a public log. The invariant was stated, applied to one branch, and left off the other. When this guard fires it does the thing it was built to prevent. + +**The `secret` table is exempted by an inline literal.** `scanned_sections` filters `_TABLE_ROLE` for `CONSUMED` and then subtracts `secret` by name, two lines under a comment claiming the scope comes from the declared partition and not from an inline list, and with no reason recorded anywhere. Measured against views-appwrite's current registry: `[secret]` has seven rows and none carries a `value`, so the clause removes zero entries today and the `isinstance(..., str)` filter on the next line already does its work. The day a secret row gains any value, the one table class whose leak matters most is silently outside the scan. + +**The rotation proof checks one side.** `assert "value:" in changed[canary] and "at-the-pin" not in changed[canary]` asserts the *pinned* value is absent. The freshly rotated value — the more damaging one, and the one the fixture already names — is never asserted absent. A regression that digests one side and interpolates the other passes this proof. + +The three share a cause: the no-print rule lives in prose and in one implementation, and nothing asserts it about the guard as a whole. + +**Partial mitigation 2026-08-12 (#242) — the leak is closed; the scan's scope is not.** + +*(The heading matters. An earlier draft of this paragraph said "Mitigated", which is neither of the two phrasings this register declares — `Mitigation — landed` and `Partial mitigation`, enforced by `tests/test_register_integrity.py`. Inventing a third phrasing passes that guard by evading its string rather than by complying, which is the identical escape ADR-014 §5 records C-15 making. This entry is partial by its own next sentence.)* + +Two of the three are fixed, and the third moved: + +1. **Both branches name the coordinate, never the value.** Each builds its own sentence — two f-strings, deliberately not one shared formatter — and neither has the value in hand. They name **all** coordinates declaring that value: measured, two pairs share one (the prod-forecasts bucket and collection share both id and name), so a `value -> name` map would have named the wrong coordinate half the time in the message a maintainer uses to find the copy. +2. **The rotation proof asserts both sides absent**, taken from the fixture's own variables rather than from two literals a rename would quietly orphan. Mutation-proven: leaking the post-rotation side while keeping the pinned side digested fails now and **passed before** — the more damaging half, unchecked. +3. **The rule is asserted behaviourally**, by `test_the_scan_reports_a_copy_without_reprinting_it`: plant a value in a fixture tree, run the real scan, read the finished message. + +**And the third one took two attempts, which is the part worth recording.** The first version asserted that every finding was *routed* through `_report_a_copy` — an AST walk over the scan's own source. Five independent reviewers were run against it and **routing turned out not to be safety**: the value could be smuggled through either of that helper's two parameters, appended with `extend` or `+=`, or reported from a renamed accumulator. Six of eight mutations survived, and one legitimate refactor *failed* it — a guard that misses the thing and fires on the innocent, which is C-82's shape and ADR-014 §3's deletion criterion at once. The docstring's claim that a caller "cannot print one however it is written" was false when written. + +Reading the finished message instead makes the whole class unreachable: it does not matter how a finding is built, which branch builds it, or what the final assertion interpolates. All five surviving mutations are now caught, including one that leaked the entire ban-set through the assertion message rather than through a finding. + +**A third pass, from an adversarial mutation review that ran 14 mutations against the second.** Twelve survived. Seven were leak channels the message test could not see, and all seven are now closed: + +- `print()` or `logging.warning()` beside the append — the message stayed clean while pytest published the value under *Captured stdout* / *Captured log call*. The behavioural test now reads `capsys` and `caplog` too. A stray debug print is an ordinary accident, not an adversarial one. +- **`_describe_changes`'s rotation proof was one-sided** — it asserted only the pinned value absent, never the rotated one, which is the more damaging half. Now both, taken from the fixture's own variables. + +**A malformed package module published itself, and the first fix for it did not work.** `ast.parse` raises with the source as its own frame's argument, and pytest renders that in full — so a package file that both fails to parse and carries a value printed itself. This entry previously said *"it takes a `Path` now and reads inside"*; **measured, that changes nothing**, because the text still reaches `ast.parse`. The fix is a shared `_parsed(path)` that refuses by path and line with `raise ... from None` — `from None` is the load-bearing part, since the chained `SyntaxError` carries the same text. Applied at **both** parse sites: fixing one and not the other is this entry's own shape. + +**A fourth pass deleted most of the third, and that is the entry's real lesson.** Four commits and five reviews had grown this file by **+195 lines**, of which **+95 was prose restating this register** and **every one of the 149 new function-body lines was a test of another test in the same file**. Two of those additions were themselves defective: + +- `test_the_drift_report_never_carries_a_value_in_any_recoverable_form` claimed to drive `_describe_changes`'s `appeared`/`removed` branches. It did not: those branch on `was is None`, and the fixtures passed `_ABSENT`, a sentinel object. It ran one branch three times while asserting three — an ADR-014 §1 defect inside the fix for an ADR-014 §1 defect — and the branches it named return constant strings and cannot leak. **Deleted.** +- `_carries`, an eight-character-window "recoverability" predicate, is **strictly worse than plain `in`**: the window degrades to exact match below eight characters, and this package's two directory names are five-character declared values. Measured, `_carries("views_postprocessing/unfao/leaky.py …", "unfao")` is `True` — it condemns a bare, safe file path while claiming a leak. **Deleted**; both call sites use plain `not in`, the vocabulary already at `test_the_environment_refusal_logs_names_and_never_values`. + +Net **−140 lines**, leaving the file **+55 over its pre-story size** rather than +195. What the +55 buys: one behavioural guard, the rotation proof's second side, the value→coordinate map, and the shared parse refusal. + +**What is deliberately NOT chased.** Four surviving mutations narrow the scan's *scope* — a length floor, a dropped section, a swallowed `SyntaxError`, a `break` after the first finding — and none is visible to a test that plants its own fixture. They are **#243**'s subject and are routed there. A fifth deletes the no-print assertion itself: infinite regress, carried here instead. Two residuals stand: the markdown branch's output is not behaviourally proven (it holds no value by construction), and a leak shorter than the planted fixture would pass. Chasing either is the whack-a-mole this epic exists to refuse. + +4. **The scan's own docstring carried four registry values**, and pytest prints the failing function's source — so the guard would have published them on exactly the event it exists to catch. The message was clean; the traceback was not. Now it names coordinates. **The wider finding is registered separately as C-97**: 33 standalone values sit in docstrings and comments across nine files, production modules included, and the AST scan excludes docstrings by a deliberate C-57 decision taken before anyone counted them. + +**Deliberately still open, and moved rather than closed:** the `secret` exemption at `:1042` and the ban-set's package-name collision are the *scope* of the scan, not its reporting, and belong with the matcher rewrite in **#243**. This entry stays open until they land, because closing it now would close a Tier 2 on two-thirds of its content. + +**Two deferrals, both with triggers (ADR-014 §4).** + +1. **The leak guards belong in `tests/test_redaction_guard.py`**, which already owns "what stops us leaking" and which `test_the_environment_refusal_logs_names_and_never_values` already points readers to. The concern is currently split across two files with no shared vocabulary. Not moved here, because moving code while fixing bugs in it is how the next defect arrives. **Trigger: when #243 finishes touching `tests/test_env_declaration.py`.** Owner: this repository. +2. **`test_the_drift_check_would_catch_a_rename` re-types its subject's comparison inline** rather than calling it, so blanking that subject's assertions leaves the proof green — the same defect its two siblings had repaired. Pre-existing, found while reading for this change. **Routed to #243**, which is already in this file. + +Cross-refs: **C-57** (the scan's own entry and its history), **C-93** (why the author's own proof did not find this), **C-90** (the sibling defect in the drift checks), ADR-014 §1, issues #242, #243. + +--- + +### C-90: A mutation proof that cannot fail, and the untested function a module was extracted to create + +| Field | Value | +|-------|-------| +| ID | C-90 | +| Tier | 2 — this is the entry PR #239 was written to close, reopened by the code that closed it. It reinstates release-blocking false alarms on the path that is this project's production release, and it does so under a docstring saying the opposite. | +| Source | `/code-review max` on PR #239 post-merge, 2026-08-11; verified by direct measurement against views-appwrite `origin/main` | +| Trigger | **Both remaining halves are proof defects, not runtime ones.** (a) Someone mutates `_unclassified_tables` or `_TABLE_ROLE` and believes the tautological proof covers it. (b) Someone changes `registry_current` — the reason `tests/seam_registry.py` exists — and the suite stays green. *(The original trigger, an unrelated coordinate arriving upstream, died with `arrived` in #245.)* | +| Owner | This repository. | +| Location | `tests/test_env_declaration.py` — `test_the_table_partition_would_catch_a_new_table_and_a_vanished_one` (the tautology); `tests/seam_registry.py::registry_current` (untested). Function names, not line numbers: this entry has cited stale ones before. | + +**~~`arrived` is not filtered by the names this package reads.~~ RESOLVED 2026-08-12 (#245) — deleted; see the mitigation below. Left visible because the reasoning it prompted is the entry's most useful part.** `changed` is; `arrived` is computed over every row of every table this package depends on. Measured on the live registry: **25 rows, 8 of which this package never reads** — six of them keys and callers belonging to other repositories. So views-appwrite issuing one more key for an unrelated repo turns both partner parametrizations red here, with a message demanding a `SEAM_CONTRACT` re-pin for a coordinate this package cannot use. + +That is the exact failure class C-86 records and that PR #239 was written to remove, and the same test's docstring seven lines above says **"Silent through: prose edits, `[meta]` bumps, and rows belonging to anyone else."** The prose describes the check that was designed; the code implements a wider one. + +There is a real question underneath, and it should be decided rather than inherited: a *new* coordinate in a table we read may be one we must adopt. That argues for table-granularity on arrival and row-granularity on change. **They cannot both stand.** + +**DECIDED 2026-08-12 (#245): the docstring won, and the cost is real.** The arrival half was deleted. A coordinate views-appwrite issues *for this package* — or a second `[contract.*]` row for a future partner such as views-productionapi — is now **silent** until a human reads the registry: no test, no run-time assert, nothing. `assert_env_declared` cannot see it, because it iterates the names this package already declares. That is the accepted price of not being reddened by every unrelated row, and it is recorded here rather than left to be discovered. *(The half's own defence — that `[contract.*]` "arrived exactly this way and nothing else here would have seen it" — was false at table granularity, where the partition check catches it, and true at row granularity, which is exactly the cost now accepted.)* + +**The partition's mutation proof cannot fail.** `assert not _unclassified_tables(base)` where `base = {name: {} for name in _TABLE_ROLE}` reduces to `set(_TABLE_ROLE) - set(_TABLE_ROLE)`, empty for every possible input. Its message — *"the real registry's tables must all classify"* — asserts a fact about a file this test never opens. It is decoration inside the test whose own docstring is about removing decoration. + +**`registry_current` has no test.** The module `tests/seam_registry.py` was extracted for one reason: two copies of the reader disagreed about whether to read the sibling's `main` or its working tree, and reading the working tree is issue #196 verbatim. The function that settles it is called by five tests and is the subject of none. Replacing its body with `rev-parse HEAD` — the defect it exists to prevent — leaves the suite at its exact baseline. Three of its error branches are executed by nothing. + +**Partial mitigation 2026-08-12 (#245) — the false-alarm half is gone; the two proof defects are not.** + +`arrived` is **deleted**. Measured before deleting: each partner reads 13 of the 25 rows in the tables this package depends on, and 8 of those rows belong to no repository here — so the check subscribed this repository to another repo's changelog. Mutation-proven after: an unrelated API key and a third partner's contract row are silent; a rotation and a removal still fire. The docstring and the code now agree, and the stopping rule sits above the check. + +**Still open, and routed to #246**: the partition's mutation proof is a tautology (`base` is built from `_TABLE_ROLE`, so `_unclassified_tables(base)` is empty for every possible input), and `registry_current` — the function `tests/seam_registry.py` was extracted to create — has no test, so replacing it with a working-tree read leaves the suite at baseline. This entry stays open until both land. + +Cross-refs: **C-86** (whose partial-mitigation paragraph this falsifies), **C-89** (the sibling defect in the no-copy scan), **C-93**, **C-91**, ADR-014 §1/§2, issue #196. + +--- + +### C-91: The git plumbing this arc added turns ordinary developer states into hard errors, bare tracebacks, and one possible hang + +| Field | Value | +|-------|-------| +| ID | C-91 | +| Tier | 3 — no wrong data and nothing silent. It taxes every contributor who does not already have the exact sibling checkout this repository assumes, and it does so with diagnoses that point at the wrong cause. | +| Source | `/code-review max` on PR #239 post-merge, 2026-08-11 | +| Trigger | Any of: a contributor clones views-appwrite shallow, single-branch, or before the pinned commit; a table classified `CONSUMED` upstream is written as flat keys rather than sub-tables; a contributor has `commit.gpgsign` or a global `core.hooksPath` set. | +| Owner | This repository. | +| Location | `tests/seam_registry.py:71` (refusal diagnoses), `:150` (`rows`), `tests/test_env_declaration.py:820` (the scratch repo). | + +**A stale clone produces four errors carrying the wrong explanation.** The reader the extraction replaced read the file off disk, so an older checkout simply read an older file. Now a clone that predates the pinned commit — or is shallow, or was made `--single-branch`, which matters because views-appwrite's default branch is not `main` — raises *"does not resolve to a commit … an empty ref reads the index and a branch reads a moving tip"*. That names neither cause and does not say `git fetch`. Meanwhile `test_the_pinned_commit_is_reachable_from_the_contract_repos_main` detects the identical root cause and *skips* with the right remedy. One condition, one skip, four errors, three explanations. + +**`rows()` raises a bare `AttributeError` on a shape the live registry already has.** It guards a null section and not a scalar row. Verified: views-appwrite's `[test_environment]` holds `status` and `fact` as top-level strings. That table is `IGNORED`, so nothing breaks today — but when the partition check fires on a new upstream table, its own message instructs the maintainer to classify it `CONSUMED` or `MIRRORED`, and doing so for a table written that way returns a traceback pointing into a dict comprehension. From the module whose docstring says a helper justified by failing legibly must not hand back a bare traceback. + +**The scratch repo inherits the developer's global git config and has no timeout.** `test_the_pinned_reader_refuses_every_way_a_baseline_can_be_wrong` sets `user.name` and `user.email` and stops. With `commit.gpgsign = true` it fails with a bare `CalledProcessError` — `capture_output=True` swallows git's explanation. With a passphrase-protected key it blocks on pinentry with no `timeout`, hanging the whole run; `conftest.git_output`, which this helper bypasses, caps at 30 seconds. The leak was anticipated for identity and not for the setting that blocks. + +Cross-refs: **C-90** (the same module's untested core), **C-88** (why the module exists outside `conftest.py`), ADR-008 (explicit failure), issue #196. + +--- + +### C-92: Nothing here checks that a consumer SELECTS by the delivery label — for either partner + +| Field | Value | +|-------|-------| +| ID | C-92 | +| Tier | 2 — the failure mode is invisible by construction and is on the live FAO path. Upload succeeds, storage is billed, the consumer's endpoint returns empty, nothing raises anywhere. | +| Source | `/code-review max` on PR #239 post-merge, 2026-08-11 | +| Trigger | **Either consumer** changes how it selects — to a category, a metadata field, a query builder — without touching its served-name constant. Also live: `manager.py:117` in both drops the name filter entirely when `model_name` is falsy. | +| Owner | Shared: views-faoapi owns the check; this repository owns noticing it does not exist. | +| Location | `views_postprocessing//product.py::CONSUMER_DOCUMENT_NAME`; `tests/test_product.py::test_the_declared_consumer_name_matches_the_registry` (what remains); ADR-017 §5/§7. Function names, not line numbers. | + +PR #239 retired the FAO half of the source-reading check on the strength of views-faoapi#379. That was the right sequencing — §5 requires the consumer-side check to land first, and it had. **But the two checks are not the same check.** + +views-faoapi#379 binds their *served-name constant* to the registry row. The assertion this repository deleted was `'filters["name"] = self.model_path.model_name'` — that they still *query* on it. So views-faoapi can refactor its selection mechanism, leave its name constant untouched, pass #379, pass our registry comparison, and serve an empty endpoint. The deleted assertion carried that exact sentence: *"the name may still match while the consumer filters on something else entirely — same invisibility, different cause."* Nothing carries it now. + +**Verified 2026-08-12, and the state is good — which is why this is a risk and not an incident.** Read at `views-faoapi@origin/development`: `_CONSUMER_DOCUMENT_NAME = "un_fao"` (`src/views_faoapi/managers/api.py:59`) reaches `APIPathManager(...)` at `:1282`, and `managers/prediction/manager.py` still filters on it at `:117` and `:435`. Their D2 test — views-faoapi's `tests/test_seam_contract_binding.py` — imports the constant from the production module rather than re-typing it, which is better than it had to be. **The composition is what nothing asserts**: constant↔registry is checked by them, query↔constant was checked by us and is not any more. Also worth recording: their D2 check is on `development`; their `main` is 22 commits behind at PR #357, so a reader taking "#379 merged" to mean "live on their default branch" is over-reading it. + +**C-87 is not this.** C-87 records that we verify our copy against the declaration rather than the consumer's code against it. This is narrower and worse: for one partner we briefly had the second check and gave it up for something that does not cover the same failure. + +**~~A second, structural half.~~ MOOT 2026-08-12 (#248)** — `_CONSUMER_SELF_CHECK_PENDING`, `test_the_pending_list_is_not_empty` and the `SIBLINGS` note it describes were all deleted with the mechanism. Left visible because the reasoning still applies to any future map of this shape: `_CONSUMER_SELF_CHECK_PENDING` is asserted non-empty and asserted to name only real partners — never asserted to *cover* them. Every other partner map here is two-sided against `PARTNER_PACKAGES`; this one is not, so a third partner is silently exempt from the source read the day it lands. And `SIBLINGS["views-crafdapi"].note` says the fetch "buys nothing and should be removed" once the registry read exists — which PR #239 landed — while ADR-017 §5 forbids retiring that partner's source read until views-crafdapi#53 lands. A maintainer following the note does the thing the ADR forbids, and `test_the_pending_list_is_not_empty` does not object because the map stays non-empty. + +**Resolved as far as it can be here, 2026-08-12 (#248) — the check is gone, the gap is permanent, and the ask is filed.** + +Both source reads are deleted. Not because the sequencing constraint was satisfied — it **dissolved**: the reads broke twice in twenty-four hours, views-faoapi on 11 August and views-crafdapi on the 12th, each time because that repository refactored a literal argument into a named constant. Their code got better and our test went red. ADR-017 §7 said we were never entitled to depend on another repository's file layout; two breakages in a day is the evidence, and repairing the regex a third time would have been repairing the wrong thing. ADR-017 §5 carries the erratum. + +**The gap is now permanent and unguarded here, by choice.** Nothing in this repository verifies that a consumer's query uses the name it declares. The chain reads: + +| link | owner | held by | +|---|---|---| +| we upload with name N | us | construction | +| N == the registry row | us | `test_product.py::test_the_declared_consumer_name_matches_the_registry` | +| their constant == the registry row | them | views-faoapi's `tests/test_seam_contract_binding.py`; views-crafdapi's equivalent | +| **their query == their constant** | **them** | **nothing** | + +**Filed where the fact lives:** views-faoapi#390 (under their seam-verification epic #383, whose flagship this is) and views-crafdapi#55. Both carry `file:line` evidence, both note that `manager.py:117` applies the name filter *conditionally* so a falsy name broadens the query rather than failing, and both say plainly that we are not prescribing their internals. + +**Verified intact at the time of writing** — the constant reaches the path manager and the manager still filters on it, in both consumers. This is a risk, not an incident. + +**Trigger** is now theirs to clear and ours to notice: when either issue lands, this entry closes for that partner. Until then the honest statement is that a delivery is verified by two values this platform authored agreeing with each other, plus the consumer's own word. + +Cross-refs: **C-87** (the broader residual this sharpens, not duplicates), **C-94** (the producer-side preflight that would close it without asking anyone), ADR-017 §5/§7/§8/Appendix B, views-faoapi#390, views-crafdapi#55. + +--- + +### C-88: The platform declarations live in pytest's fixture file, so nothing outside the test tree can reach them + +| Field | Value | +|-------|-------| +| ID | C-88 | +| Tier | 3 — no correctness risk and nothing silent. It is a boundary that has already forced one duplication for structural rather than design reasons, and it will force the next one the same way. | +| Source | `falsify` against the SOLID / component-principle lens, 2026-08-11 | +| Trigger | **Either.** (a) A second non-test consumer needs one of these declarations and has to copy it. (b) A fifth declaration is added to `tests/conftest.py` — the file is at four, and the threshold for "dumping ground" is not a number but the moment nobody can say in one sentence what the file is for. | +| Owner | Whoever adds the next declaration, or the next non-test consumer. Not urgent; it gets more expensive slowly. | +| Location | `tests/conftest.py` (grew again this week; `wc -l` for the number, which moves); `scripts/build_gaul_lookup.py:73`; ADR-016 §4. | + +`tests/conftest.py` is pytest's fixture file. It currently holds **four unrelated groups**: the package taxonomy (`PARTNER_PACKAGES`, `MACHINERY_PACKAGES`), the sibling repositories (`Sibling`, `SIBLINGS`, and three resolver functions), the consumer mapping (`CONSUMER_REPO`), and two git helpers (`git_output`, `commit_is_on_main`). None of those is a test fixture. They are declarations about the platform that happen to be consumed by tests. + +**The concrete cost, which has already been paid once.** `scripts/build_gaul_lookup.py` needs the same sibling-location fact and cannot have it: *"a script must not import from `tests/` — that is the dependency direction backwards."* So it hardcodes `"VIEWS_DATAFACTORY"` at `:73` while `SIBLINGS` declares the same string in `conftest.py`. + +That duplication is **defended on WET grounds and the defence is sound** — the two contracts genuinely differ (the script returns a `Path` even when the checkout is absent so it can raise its own message; the test helper returns `None` because a missing sibling is a normal skip), and a guard asserts the two resolve to the same place. This entry does not ask for that to be merged. + +**What it records is that the choice was not free.** The script could not have reused the declaration even if reuse had been right, because of where the declaration lives. A structural constraint and a design decision reached the same answer, and only one of them was examined. + +**Why this is Tier 3 and not higher.** Nothing is wrong today. Every guard works, the duplication is guarded, and moving the declarations would touch a dozen imports for no immediate gain. The risk is the slope: `conftest.py` is where a declaration goes when nobody asks where it belongs, and each addition makes the next one more natural. + +**What "fixed" would look like**, when the trigger fires: a small module that owns the platform declarations — importable by tests, scripts and, if ever needed, package code — with `conftest.py` reduced to what pytest actually needs from it. That is the shape, not a commitment; the point of the trigger is that the second incident tells you whether it is right. + +Cross-refs: **C-46** (which records the builder's separate resolver and the guard that they agree), ADR-016 §4, ADR-002 (dependency direction). + +--- + +### C-86: The release path now depends on two other repositories, and there is no way past a red build + +| Field | Value | +|-------|-------| +| ID | C-86 | +| Tier | 2 — no wrong data ships and nothing is silent. What is at risk is the ability to *ship at all* on a day when someone else's repository has moved, on the path that is this project's production release. Latent today, live the moment a status check becomes required. | +| Source | External review of ADR-016 by the views-appwrite and views-faoapi seats (#231, #233), 2026-08-10 | +| Trigger | **Two, and the first is the one to watch.** (a) A status check becomes required on `main` — at that moment this stops being latent. (b) An upstream merge to a sibling's `main` reddens this repository while a delivery fix is waiting. | +| Owner | Simon. Both available responses are console actions: add a bypass actor to `protect_main`, or accept the coupling as written. | +| Location | `.github/workflows/run_pytest.yml` — the two sibling checkout steps; ADR-016 §7, §7a, §7b. | + +ADR-016 has CI check out `views-appwrite` and `views-crafdapi` so that cross-repository checks run on every change rather than on a maintainer's habits. That is the right trade and the entry does not dispute it. What it records is the cost, which was accepted in the ADR on a justification that turned out to be false. + +**The false justification.** ADR-016 §7 originally said the maintainer could merge over a failing check when something was urgent. Two reviewers challenged it independently. Measured 2026-08-10: `protect_main` lists **zero bypass actors**, and a GitHub ruleset applies to everyone except the actors it names — so administrator status confers no exemption. There is no classic branch protection either, so no `enforce_admins` route. The claim is withdrawn in the ADR; the risk it papered over is this entry. + +**Why this is latent rather than live.** `protect_main` currently requires **no status check at all** (C-81's enforcement half). So today a red build blocks nothing and this coupling costs nothing. The instant a required check is added — which C-81 asks for, correctly — the coupling becomes real and unbypassable in the same change. **Two open items that each look independently sensible combine into something neither of them says.** + +**The rate is not hypothetical.** views-appwrite reports five registry editions in four days (v1.4.0 2026-08-02 through v1.4.4 2026-08-05), **four of them observation-driven** — recording console facts, correcting a key's scopes — carrying no obligation for any consumer. Each would have reddened this repository and blocked a release. + +**What would resolve it, in order of preference:** views-appwrite#76 makes the obligation-carrying distinction machine-readable, so observation-only bumps stop firing the check at all — filed, and that seat volunteered it. Failing that, a bypass actor restores the escape. Failing both, the coupling stands as ADR-016 §7 describes, which is defensible but should be chosen rather than discovered. *(⚠ "filed" is stale as of 2026-08-11 — #76 has **landed**, as registry v1.6.0. See the amendment at the end of this entry; the sentence is left as written because what it asked for and what arrived are worth comparing.)* + +Cross-refs: **C-81** (the enforcement half, whose fix activates this), **C-46** (whose recommendation against per-PR sibling checkouts this overrode, with the reasoning recorded there), ADR-016 §7/§7a/§7b, views-appwrite#76. + +**Partial mitigation 2026-08-11 — the false-alarm rate, not the coupling.** The check that +kept firing compared registry *version strings*. It has been replaced by three that match +on the facts this repository actually declares: the pinned commit must declare the pinned +version; every top-level table upstream must be classified here; and no row we read may +differ between the pinned edition and the current one. + +Measured across the window that prompted this entry — v1.4.4 → v1.5.2, three editions in a +week — the new checks are **green**, because none of those editions touched a row this +package reads. Under the old check every one of them was a red build blocking a release. + +**This does not close the entry, and the distinction matters.** C-86 is about CI depending +on two other repositories with no way past a red build. That dependency is untouched: the +sibling checkout still happens, `protect_main` still has zero bypass actors, and a change +upstream that *does* touch a row we read will still redden this repository and block a +merge — correctly, and that is the point. What changed is that it now fires for reasons +that carry an obligation. + +The entry's trigger is unchanged and remains a console action: **a status check becomes +required on `main`**, at which moment the coupling stops being latent. + +**Amended 2026-08-11 — the first-preference resolution has LANDED, and this entry found out +from a guard rather than from a notification.** views-appwrite#76 shipped as registry +**v1.6.0**: a new `[edition."x.y.z"]` table marking each edition `obliges_consumers = +true|false`, which is exactly the machine-readable distinction the paragraph above asks for. +The sentence *"filed, and that seat volunteered it"* was true when written and is now stale. + +Two things are worth recording about how it arrived. The partition check added the same day +went red on it unprompted, on its first live encounter with an upstream table nobody here +had classified — which is the whole reason that check is directional. And the row-level +drift check stayed **correctly silent**, because a new table this package does not read is +not drift in anything it depends on. The two behaved exactly as designed on data neither +was tested against. + +**Adoption is a follow-up, not part of the change that noticed it.** `[edition]` is +classified `IGNORED` in `tests/test_env_declaration.py::_TABLE_ROLE` — declared, not +silently unseen. Nothing here reads `obliges_consumers` yet, so the false-alarm rate is +still carried by the row-level differential rather than by upstream's own flag. + +**Deferral, with the trigger ADR-014 §4 requires** — this was carried in a pull-request +description, where deferrals go to be forgotten: + +- **What:** re-pin `SEAM_CONTRACT_VERSION` / `SEAM_CONTRACT_COMMIT` in both + `views_postprocessing/{unfao,crafd}/appwrite_env.py` (currently v1.5.2 / `c7b597e`), and + read `[edition].obliges_consumers` so an observation-only edition cannot fire anything. +- **Trigger — both halves have now FIRED:** the partition fired on v1.6.0 (2026-08-11), and + views-appwrite#76 landed. The re-pin is therefore **due**, not deferred; what remains + deferred is reading the new flag. +- **Owner:** whoever next touches an `appwrite_env.py`. Re-pinning is hygiene with no + safety consequence now that the differential exists — which is precisely why it needs a + written trigger rather than a good intention. + +Cross-refs for this amendment: **C-57** (the drift detector this rides on), ADR-016 §7a/§7b, +views-appwrite#76 (**delivered**, registry v1.6.0). + +**⚠ CORRECTED 2026-08-12, and PARTLY RESOLVED the same day (#245).** The paragraph +beginning *"Partial mitigation 2026-08-11"* claimed the replacement checks "match on the +facts this repository actually declares". **Two of the three did not**, and an earlier +draft of this correction said one — the review that caught it is the reason this paragraph +is longer than it wants to be. + +**The first, now fixed.** The drift check's arrival half ran over every row of every table +this package depends on, with no filter for the names it reads. Measured on the live +registry: each partner reads **13 of 25** rows, and **6 belong to no repository here** — +three caller keys for other consumers, and three platform key slots still marked +`status = "planned"`, one of which names *"un_fao delivery"* among the identities it would +be issued for. C-90's original wording was "belong to no repository here"; an earlier draft +of this paragraph upgraded it to "other repositories entirely", which is the same +overstatement this paragraph exists to correct, two sentences from correcting it. The other two rows this package does not +read are its own delivery labels — both declare `producer = "views-postprocessing"` — and +they are covered by `tests/test_product.py`, not by this check. *An earlier draft of this +paragraph said "8 belong to no repository here", which upgraded a careful claim in C-90 +into a false one and propagated it to four places.* + +**And it never actually fired in anger.** Measured: **zero rows have arrived in the +depended-on tables since the pin** — the arrival half was green on every real edition it +ever saw, from its introduction on 2026-08-11 to its deletion on 2026-08-12. An earlier +draft said an upstream key "reddened this repository and blocked a release". That is a +mutation result written in the past tense. What is true: a mutation shows it *would* fire, +and it *would* block once C-81's required check lands — `protect_main` today carries +`deletion, non_fast_forward, pull_request` and no required status check, which is this +entry's own "latent rather than live" paragraph. + +**The arrival half is now deleted** (#245). The drift check fires on exactly one condition — +a row this partner declares differs between the pinned edition and the sibling's `main` — +and that stopping rule is written above the check itself. Its self-defence turned out to be +false: it claimed `[contract.*]` *"arrived exactly this way and nothing else here would have +seen it"*, but `[contract]` is a top-level **table**, caught by the partition check directly +above it. + +Mutation-proven both ways against the live registry: an unrelated API key and a third +partner's contract row are now **silent**; a rotation and a removal of a row this partner +reads still **fire**. A new test asserts the silence direction, so a third widening meets +an objection rather than a paragraph. + +*(That silence test needed two attempts, and the first is worth recording because it is +this entry's own disease. It called `_describe_changes` directly — the helper underneath +the check — and was worthless: re-adding the deleted arrival half to the real check left +the whole suite green, because that half never lived in the helper the test was asking. +The "mutation proof" offered for it was invalid too: it mutated the test rather than the +code, which proves nothing. It now drives the real check through monkeypatched readers, +and re-adding the arrival half fails it. **A guard must be pointed at the thing it claims +to guard**, which is the sentence this whole entry keeps re-learning.)* + +**The second inaccuracy stands, and is this entry's remaining rate risk.** *"Every +top-level table upstream must be classified here"* fires on any new table regardless of +whether this repository declares anything about it — and **it has already fired for exactly +that reason**: `[edition]` arrived at v1.6.0, an edition the registry's own `[meta]` calls +*"additive and opt-in; obliges nobody"*, and it reddened this repository. That event is +recorded **earlier** in this entry — the partition "went red on it unprompted", "which is +the whole reason that check is directional" — and that record is true of the *detection* +and silent about the *cost*. Adopting `[edition].obliges_consumers` is the deferral that +would fix it, and its trigger is below. + + +--- + +### C-87: The delivery label will be checked against a declaration, and nothing will check the declaration against the consumer + +| Field | Value | +|-------|-------| +| ID | C-87 | +| Tier | 2 — the failure mode is invisible by construction. The upload succeeds, the storage is paid for, the consumer's endpoint returns empty, and nothing anywhere raises. That is the shape ADR-013 §4.1a calls *"invisible to the consumer, not merely degraded"*. | +| Source | ADR-017 §8, sharpened by external review (#232, #234), 2026-08-10 | +| Trigger | **Either half going missing.** (a) ~~`views-faoapi#379` or `views-crafdapi#53` is closed without the check being written.~~ **Both closed WITH the check written** (2026-08-11, 2026-08-12). The live trigger is now (a') either consumer's *query* stops using its declared constant — see **C-92**, and views-faoapi#390 / views-crafdapi#55. (b) A private API operated by a **third party** becomes a consumer — at which point the second half cannot be required at all and this becomes permanent. | +| Owner | The consumer-side seats own the check; this repository owns noticing that it exists. | +| Location | `views_postprocessing//product.py::CONSUMER_DOCUMENT_NAME`; the check in `tests/test_product.py`; ADR-017 §5, §8, Appendix B. | + +ADR-017 decides that the delivery label is declared in the public coordinate registry and that each side verifies **itself** against that declaration, so neither repository reads the other's source. That is the right rule and this entry does not dispute it. + +Its residual is stated plainly in §8 and belongs here rather than only in a document: **we will verify our copy against the declaration, not the consumer's code against it.** If a consumer quietly starts filtering on something else, our check passes and the delivery is invisible exactly as before. + +**Why this is a risk and not merely a note.** The second half is real work in repositories this project does not control. `views-faoapi#379` has a willing owner. `views-crafdapi#53` is blocked on that partner's data contract and could sit for a long time. Both have now landed. The label's agreement with reality **no longer rests on any source-reading check**: both were deleted on 2026-08-12, not because those gates opened but because a consumer improving its own code broke the mechanism twice in a day (ADR-017 §5 erratum). What remains is C-92 — a consumer's word that its query uses the name it declares. + +**What was already prevented, and then accepted.** A reviewer caught that the obvious sequence created a window where the source-reading check was deleted before the consumer-side check existed, leaving a green build proving only that two values this platform authored agreed with each other. ADR-017 forbade that ordering, and the ordering was in fact honoured — both partners' consumer-side checks landed before their source-read was removed. The window exists anyway, permanently, because the source-reads were then deleted on their own demerits. This entry exists so that history has a home outside the document that states it. + +Cross-refs: **C-77** (the same field's producer-side half, resolved), ADR-013 §4.1a, ADR-017 §5/§8/Appendix B, views-appwrite#75, views-faoapi#379, views-crafdapi#53. + +**⚠ AMENDED 2026-08-12 — one partner's residual got worse, not better.** This entry says the label's agreement with reality "rests on the source-reading check … which someone could remove believing the registry check replaced it." PR #239 removed the FAO half on the strength of views-faoapi#379 — correct sequencing under ADR-017 §5, and still the wrong outcome, because **#379 and the deleted check do not cover the same failure.** #379 binds their served-name constant to the registry; the deleted assertion was that they still *query* on it. Registered separately as **C-92**, because it is narrower and more acute than the residual recorded here: not "we never checked the consumer's code" but "we checked it, for one partner, and stopped." + +--- + +### C-85: A cross-repo ask is adopted on its stated terms without anyone measuring the current state + +| Field | Value | +|-------|-------| +| ID | C-85 | +| Tier | 3 — no delivery is affected and no wrong data ships. What is at risk is spending a coordinated three-repository change on work that is already done, which is expensive in exactly the currency this platform has least of. | +| Source | #133 execution (2026-08-05) — the ask was re-measured before implementing it | +| Trigger | The next time an issue filed by another repository's seat is picked up for implementation. Before writing code, check what this repository already delivers and what the consumer already reads — the two are stated in the issue and were both wrong last time. | +| Owner | Whoever implements a cross-repo issue. This is a habit, not a mechanism; see below for why no guard is proposed. | +| Location | Not a code defect. #133; ADR-013 §2.2a. | + +#133 asked for three declared fields — `maturity`, `source`, a required schema version — and said `_save_contract` ships the forecast run only. It was filed in good faith by the views-faoapi seat, accepted, and carried on the backlog for weeks. Measured on 2026-08-05, before writing anything: + +| the ask | the measured state | +|---|---| +| stamp `source`, *"`source="unknown"` is what's live today"* | **already delivered** as `provenance.ensemble`, and views-faoapi reads exactly that key | +| declare a required schema version | **already delivered** as `contract_version` — on the run manifest and in every shard header | +| stamp `maturity` on the run manifest | genuinely missing — but it belongs in the **shard header**, which is where the consumer reads it, and it is **not this repository's to stamp** | +| *"the global historical is not uploaded"* → decide whether FAO stops receiving it | **false.** `_save_contract` uploads it, `category="historical"`, under the same interlock as the forecast. There was no decision to make | + +**Two of three fields already shipped, and the decision had no premise.** Had the issue been implemented as written, the cost would have been a `contract_version` bump — which is written *inside* the header bytes that §10 pins — and therefore a rebuild of the golden fixture and a coordinated re-vendor across all three implementing repositories, in order to add two fields that were already there. + +**Why the request was wrong is more useful than that it was wrong.** It was not careless. It was written against the **run manifest**, which is the artifact whose name suggests it carries run-level facts. The consumer reads them from the **shard header's `provenance`**, because that is where the producing pipeline's identity travels. Both seats were describing a real need and neither was describing the same object. A cross-repo ask names an artifact in the other repo's vocabulary, and vocabulary is exactly what does not survive the trip. + +**No guard is proposed, deliberately.** There is no mechanical check for "is this request still true", and inventing a ceremony — a template, a checklist field — would be process theatre that decays into an unread heading. What made the difference here was reading the consumer's source before writing any, and that is a habit worth writing down rather than automating. Registered so the next person has the worked example instead of the rule. + +Cross-refs: **C-72** (the re-vendor this would have triggered, and the trigger A2 now rides on), **C-77** (the historical leg whose correctness is what makes the co-delivery premise false), ADR-013 §2.2a, ADR-014 §4 (the deferral's named trigger), #133, views-faoapi ADR-033 and its register C-169. + +--- + +### C-84: Every identity this repo delivers under dies on 2026-11-17, within 3h35m of the other + +| Field | Value | +|-------|-------| +| ID | C-84 | +| Tier | 2 — not silent. The delivery fails loudly and completely, which is the correct behaviour and also the whole problem: there is no degraded mode, no fallback identity, and the date is known in advance. A foreseeable total outage that nobody has scheduled work against is a structural risk, not an operational surprise. | +| Source | views-appwrite coordinate registry v1.4.3/v1.4.4 — operator console read, 2026-08-05 (þing-02 A3(i)) | +| Trigger | **A date, unusually — 2026-11-17.** The registry records `VIEWS Pipeline Core` expiring 12:35 and `UN FAO` 16:10 that afternoon. Act when the un_fao delivery is next scheduled within a month of it, or when anyone plans a rotation, whichever is first. | +| Owner | Simon, and only Simon — issuing and installing keys is a console action. This entry exists so the date is visible from *this* repo's planning surface rather than only from the platform's. | +| Location | `views_postprocessing/{unfao,crafd}/appwrite_env.py` — the declared coordinates; the values live in the environment and the registry, never here. | + +The FAO delivery authenticates with the `UN FAO` key. That key expires **2026-11-17 16:10**, and the platform's other key three and a half hours earlier. Read from the console rather than inferred, and recorded in the registry this repo pins. + +**Why the gap is the finding and not the dates.** Three and a half hours is not a stagger — it is close enough that the two keys cannot cover for one another under any realistic response, and both are on the same seam. After 16:10 that afternoon every identity on it is dead at once: model and ensemble writes, the un_fao delivery, the CRAF'd delivery, all preflights, and FAO's own read access. A rotation that assumes one key can carry traffic while the other is replaced has no such window. + +**What this repo can and cannot do.** It cannot rotate anything; it holds no credentials and must not (þing-01 D3). What it can do is fail early and legibly rather than mid-delivery — and it does not currently. `appwrite_env.py` validates that the declared variables are *present*, which an expired key still is. An expired key is indistinguishable from a valid one until the first request comes back unauthorised, by which point a delivery is part-way through. + +**Deliberately not fixed here, and the reason is C-84's own shape.** A preflight that checks key validity means an authenticated call at startup, and the only project to make it against is production — which þing-02 **D2** forbids for tests and this would not quite be. The honest position is that this is a *date to act on*, not a mechanism to build, and inventing a mechanism would be building the wrong thing to feel busy. Registered so the date is not discovered by an outage. + +Cross-refs: **C-81** (the same operator session's other half — branch protection and the CI token), **C-27** (no rotation mechanism for a secret value upstream), **C-57** (the pinned-registry detector, which is how this arrived here at all — it demanded the v1.4.4 bump and the bump is what surfaced the expiry), þing-02 A3(i), views-appwrite C-65 and C-66. + +--- + ### C-15: Upload metadata lacks enrichment provenance and carries test description | Field | Value | @@ -292,7 +807,7 @@ See also D-10 (handling decision), C-43 (the *value*-correctness sibling — run | Source | `expert-code-review` (2026-06-12) | | Trigger | **Fired 2026-08-03 — see the update below.** The remaining trigger is the *extraction* one, and it is now named: a **third** in-repo partner package, **or** the first bug that must be hand-patched identically in both manager files — whichever comes first. | | Owner | Whoever adds the third partner package, or hits the first double-patch. Until one of those happens the duplication is the deliberate WET position, not a task anyone is behind on. | -| Location | `views_postprocessing//managers/.py` — `_prod_forecasts_datastore`, `__datastore`, `__appwrite_config`, and the four hardcoded `os.getenv("APPWRITE__*")` literals inside the last of those; declared names in each partner's `appwrite_env.py`. **Symbols, not line numbers** — see the note under the measurement below. | +| Location | `views_postprocessing//managers/.py` — the module-level `_build_prod_forecasts_store`, `_build_partner_store`, `_partner_appwrite_config` (methods until 2026-08-05), and the four hardcoded `os.getenv("APPWRITE__*")` names inside the last of those; declared names in each partner's `appwrite_env.py`. **Symbols, not line numbers** — see the note under the measurement below. | Mitigation: a small `DeliveryProfile` (bucket/collection/database ids, category, targets) passed to the manager — one manager class, N store configs. Scheduled **after** the FAO global delivery ships (D-09); the only immediate action is deleting the commented-out config blocks at lines 80-107, which are a mis-uncomment hazard during deadline work. @@ -345,7 +860,7 @@ See also C-24 (schema contract per store), C-77 (the fourth home for partner ide | Tier | 2 | | Source | `expert-code-review` (2026-06-24) | | Trigger | **(a) Upstream change:** when pipeline-core changes `PGMDataset` / the data loader / the postprocessor base (mid-migration: their #186/#188/#161), verify the inherited surface this repo depends on still holds. **(b) Standing work item:** the input-side de-inheritance (the sink side landed — see the 2026-07-31 update) — schedule it, don't wait for a trigger. | -| Location | `views_postprocessing//managers/.py` — the `class PostProcessorManager(PostprocessorManager, ForecastingModelManager)` statement (double inheritance); `_prod_forecasts_datastore`, `__datastore`, `__appwrite_config` (inline env/AppwriteConfig/DatastoreModule); `_validate` and `_check_coverage`; the DIP sink adapter `_ContractStorePort`. **Since 2026-08-03 all of it exists twice** — `unfao` and `crafd` are the same file with the partner name changed (C-33). Symbols rather than lines, deliberately: an earlier version of this row was invalidated by a comment edit four lines long. | +| Location | `views_postprocessing//managers/.py` — the `class PostProcessorManager(PostprocessorManager, ForecastingModelManager)` statement (double inheritance); `_build_prod_forecasts_store`, `_build_partner_store`, `_partner_appwrite_config` (env/AppwriteConfig/DatastoreModule — moved OFF the class 2026-08-05, see the update below); `_validate` and `_check_coverage`; the DIP sink adapter `_ContractStorePort`. **Since 2026-08-03 all of it exists twice** — `unfao` and `crafd` are the same file with the partner name changed (C-33). Symbols rather than lines, deliberately: an earlier version of this row was invalidated by a comment edit four lines long. | `UNFAOPostProcessorManager` subclasses **two concrete** pipeline-core base classes (`PostprocessorManager`, `ForecastingModelManager`) and **interleaves infrastructure** (env reading, `AppwriteConfig` construction, `DatastoreModule`, path resolution) with the FAO **business logic** (GAUL enrichment, the 9-column null gate) inside the lifecycle hooks. Consequences: (a) the FAO logic cannot be instantiated or unit-tested without the full framework + Appwrite env + viewser; (b) **pandas cannot leave the delivery path** because the inherited data loader and `PGMDataset` are pandas — gated on pipeline-core's own DataFrame retirement; (c) **SDP exposure** — heavy *inheritance* coupling to a pipeline-core that is itself unstable (mid-migration), so upstream changes break far from their cause (cf. C-27, C-29); (d) it's the repo's only composition-over-inheritance violation. The dependency itself is correct (`unfao.py` genuinely *is* a pipeline-core postprocessor) — the issue is its **blast radius**. Mitigation (does **not** fight the Template-Method framework): keep the subclass as a **thin shell** but extract `enrich` + `validate` + the 9-column contract into a pipeline-core-free core object the manager *calls*, and wrap the Appwrite I/O behind a small delivery-sink adapter (DIP). This makes the FAO logic testable standalone and insulates it from pipeline-core churn. @@ -379,97 +894,235 @@ See also C-24 (schema contract per store), C-77 (the fourth home for partner ide **What actually changed, and what did not.** The blast radius is no longer *one file wide* — it is **one file, twice**, which is a different and slightly worse property: an upstream change now has two identical landing sites and no mechanism guarantees they are patched together (C-33). What did **not** change is the more important half: the count is still **bounded and pinned**. `test_views_pipeline_core_is_confined_to_the_partner_managers` (renamed in #211 — it had asserted *two* under a name that said *one*) was widened to an explicit allowlist, not deleted, so a *third* importer still fails CI. Every other module in the repository remains pipeline-core-free, including the whole of `contract/` and `delivery/`, and `tests/test_clone_readiness.py` still proves the machinery imports in a subprocess without it. +**Recorded as ADR-015 (2026-08-04).** The reasoning for keeping this import — and the two-part condition under which it is revisited — now lives in `docs/ADRs/015_the_pipeline_core_appwrite_import.md` rather than only in issue #146 and the platform's deliberation folder. That was the condition attached when the deferral was ratified, and it had not been met. The ADR also records that the **supply** half of the trigger has *partially* moved: pipeline-core 3.0.0 relocated provisioning and transport out of the Appwrite module (3,064 → 2,841 lines), but exports no client surface, so there is still nothing to unwind to. + **On þing-02 S24(5).** `docs/CLONING.md` cited that verdict as forbidding these imports outright. Reading it directly (`þingit/02_credential_identity_key_ownership/sáttmál.md:240-242` — precondition (5) itself; the section opens at `:232` under the heading *"§5 — The clone (`un-crafdapi`)"* — and `orð_dómr.md:418-441`), it binds *"the clone"* — `un-crafdapi` and `views-productionapi`, repositories **git-cloned from views-faoapi** — and does not reach an in-repo partner package of the producer. CLONING.md over-claimed; PR #211 corrects the citation rather than weakening the rule. This entry's own scope is unaffected: the coupling is a design concern here regardless of what the verdict binds, and issue **#146**'s deferred unwind now covers two files instead of one. -Tier held at 2. The residual scope — the double inheritance and the framework-bound instantiation — is unchanged, and is still gated on views-pipeline-core 3.0.0 (C-44/C-62). +**Update 2026-08-05 — the stated gate was wrong, and the source half of the mitigation has now landed.** + +*The gate first, because it is what kept this entry parked.* The line below said the residual was *"gated on views-pipeline-core 3.0.0, which is a release signal rather than engineering work"*. 3.0.0 shipped on 2026-08-03 and nothing became possible. Two different gates had been conflated: + +- **Consequence (b), the pandas gate** — genuinely gated on pipeline-core's DataFrame retirement, which is an epic (#186/#207), not a version number. It lifted on its own on 2026-07-27 when the frame-native fetch shipped, and 3.0.0 had nothing to do with it. +- **The de-inheritance** — never gated on any pipeline-core release. **It is gated on views-models**, which is the thing nobody had written down. `postprocessors/un_fao/main.py:27` constructs `UNFAOPostProcessorManager(...)` directly, and the framework's Template Method drives `_read`/`_transform`/`_validate`/`_save`. **The inheritance *is* the integration contract with views-models.** Removing it means writing a different launcher there, so it is a two-repository change and an operator decision — not, as this entry implied for two days, unblocked work waiting on nobody. + +*What landed.* This entry's own prescribed mitigation was to wrap the Appwrite I/O behind a DIP adapter. The **outbound** half landed in July as `_ContractStorePort`. The **inbound** half never did: `_prod_forecasts_datastore`, `__datastore` and `__appwrite_config` were still methods on the manager, constructing `AppwriteConfig` and `DatastoreModule` inline. They are now module-level functions taking declared arguments — `_build_prod_forecasts_store`, `_build_partner_store`, `_partner_appwrite_config`. + +That is a smaller change than it sounds and a bigger one than it looks. Smaller: no delivered byte moves, and the three functions are the same code with their inputs declared. Bigger: **consequence (a) is substantially discharged for this surface.** Store construction, its refusals and their ordering are now testable with no manager instance, no views-models path manager and no Appwrite environment — `tests/test_store_construction.py`, 14 tests, which could not have been written at any price a day ago. That is the first time any part of the manager seam has been reachable without the framework. + +*Two things found while in there.* `self.ensemble_path_manager` was assigned in `__init__` and in the store builder, read exactly once four lines after being set, and read nowhere else in either partner package or in views-models — a method-local value wearing the costume of manager state. And a `loa = "pgm"` assignment followed by `if not loa: raise`, an unreachable branch guarding a variable never used again, beside a commented-out block that had computed it for real. Both gone. + +*Measured.* The manager **class** went 351 → **272 lines** (16 → 14 methods); both partner files are 440 lines against a 450 directory budget. A new ratchet, `test_the_manager_class_itself_stays_thin`, bounds the class at 300 — added because the existing budget counts the *directory*, so it cannot see a class re-absorbing logic, and this refactor is precisely the shape it is blind to. Both bounds now stand; neither was relaxed. + +**Residual scope, stated honestly.** The double inheritance itself, and with it consequences (c) SDP exposure and (d) composition-over-inheritance. Not unilaterally actionable, and arguably not a defect: this entry has always conceded that *"the dependency itself is correct — `unfao.py` genuinely **is** a pipeline-core postprocessor"*, and ADR-015 §1 argues there is nothing to unwind to. What remains is exposure to an upstream that moves, which 3.0.0 demonstrated by moving. + +**Trigger, corrected.** Not a pipeline-core release. Revisit when **views-models changes how postprocessors are launched** — at which point de-inheritance becomes a one-repo change rather than two — or when an upstream base-class change actually breaks a delivery, which `tests/test_framework_contract.py` now catches here rather than on the partner's run. + +Tier held at 2: the blast-radius argument is unchanged for what remains, and containment is not removal. *Did not:* the double inheritance (the `class UNFAOPostProcessorManager(...)` statement — this row cited `unfao.py:80` when written, and that number has moved twice since) stands, and so do consequences (a) — the FAO logic still cannot be instantiated without the framework — and (c)/(d). **This entry remains open on exactly that scope.** Its remaining fix is gated on views-pipeline-core's 3.0.0 (C-44/C-62), which is a release signal rather than engineering work. -See also C-07/C-27/C-29 (pipeline-core coupling symptoms), C-39 (the dead-mapper cleanup that precedes any unfao restructuring), **#45** (the delivery-side draw carrier — ship `(N, S)` uncollapsed as a native frame, the producer half of this same problem), and **epic #85** (the migration backlog). +See also C-07/C-27/C-29 (pipeline-core coupling symptoms), C-39 (the dead-mapper cleanup that precedes any unfao restructuring), **#45** (the delivery-side draw carrier — ship `(N, S)` uncollapsed as a native frame, the producer half of this same problem), and **epic #85** (the migration backlog). + +--- + +### C-72: The pyarrow pin holds this repo inside a high-severity CVE, and the fix changes the wire bytes + +| Field | Value | +|-------|-------| +| ID | C-72 | +| Tier | 3 — **not currently exploitable here**: the CVE is a *read-path* use-after-free and this repo only **writes** arrow in production (the three `vf_arrow.load` calls are all in tests). It is Tier 3 rather than 4 because the bytes we write are read by views-faoapi under the same platform-wide pin, so the platform's exposure is real even though this repo's is not, and because the remedy collides with a ratified contract. | +| Source | `manual` (2026-08-01) — GitHub surfaced 32 Dependabot alerts on the push that made `main` current; reviewed alert by alert | +| Trigger | When lifting the `pyarrow < 17` ceiling — pipeline-core **#280** (the viewser → views-storage chain) is the platform half — **regenerate the ADR-013 §10 golden fixture in the same change and notify the faoapi seat**, because the upgrade is expected to change the delivered bytes. Also fires if this repo ever gains a production arrow **read**. | +| Location | `pyproject.toml:15` (`pyarrow = ">=16.1.0,<17.0.0"`); `poetry.lock`; the affected operation would be `views_frames.io.arrow.load`, called only at `tests/test_wire_shard.py:90`, `tests/test_wire_fixture.py:73`, `tests/test_wire_header.py:39` | + +**GHSA — "Apache Arrow: potential use-after-free when reading IPC file with pre-buffering."** Vulnerable range `>= 15.0.0, < 23.0.1`; patched in **23.0.1**. This repo's declared pin is `>=16.1.0,<17.0.0` — **entirely inside the vulnerable range, with a ceiling that excludes the fix.** It is the only one of the 32 alerts against a dependency this repo declares itself (`pyproject.toml`); the other 31 are `poetry.lock` resolution — see the disposition note below. + +**Why it is not exploitable here, stated precisely rather than reassuringly.** The defect is on the *read* path. Production writes arrow (`contract/wire/shard.py` → `views_frames.io.arrow`) and never reads it back; the only `vf_arrow.load` calls in the repository are the three byte-parity tests above, operating on committed fixtures we generated. So there is no path from partner or producer input to the vulnerable code **in this process**. + +**Where the exposure actually sits: views-faoapi.** It reads these shards, under the same platform-wide `pyarrow < 17` ceiling. Our output is its input. This entry does not claim to size that repo's risk — it records that the platform's pyarrow pin is a shared exposure and that the consumer is the reading half. + +**The collision worth understanding — the fix and the contract point in opposite directions.** The five byte-parity failures this repo has treated as "known local toolchain noise" all session are precisely this: the local environment runs **pyarrow 23.0.1 — the patched version** — while CI runs the pinned 16.1.x, and the two produce **different bytes** for the same input. So upgrading to the patched pyarrow is not a dependency bump; it **changes the §10 golden fixture**, which ADR-013 declares normative and which views-faoapi verifies against. Security remediation here is a **contract event**, not maintenance, and must be sequenced with the consumer. + +**⚠ TWO CORRECTIONS to this entry, made the same day it was written (2026-08-01), on evidence gathered while filing the cross-repo issues.** + +**(a) views-faoapi is NOT exposed, and the platform ceiling is not uniform.** This entry implied the consumer shared our pin. It does not: faoapi pins **`pyarrow==23.0.1`** — the patched version — in its `pyproject.toml:31`, and it is the repo that actually reads the wire (`forecast/ingestion/wire_reader.py` → `arrow.load`). So nobody on the platform is currently exposed through this path, and the `<17` ceiling views-pipeline-core#280 exists to lift is binding on the **producers** only. One consumer has already moved past it, in production. + +**(b) The re-vendor obligation is stronger than "our fixture changes".** ADR-013 §10 is explicit: *"All three implementing repos' test suites consume the same bytes … The other two repos **vendor a copy** and carry a **pinned root-hash equality test** … A change to the fixture is a change to the contract."* Verified — faoapi's copy is at `tests/forecast/golden/wire_contract/SHA256SUMS`, pinned by views-faoapi's `tests/forecast/test_wire_golden_fixture.py`. So the upgrade is a **coordinated three-repo re-vendor**, and it carries an undecided question: whether it bumps `contract_version` (the payload schema does not change, only the encoder's bytes — §10 says a fixture change *is* a contract change). + +**Filed, so the relocation is complete rather than assumed** (the C-08 lesson): **views-postprocessing#174** (the coordinating issue), **views-faoapi#348** (heads-up + two questions only their seat can answer), and a comment on **views-pipeline-core#280** adding the security dimension and the non-uniform-ceiling finding. + +Cross-refs: **C-62** (the transitive dependency drag; the other 31 alerts), **C-46** (the datafactory version-state coupling), ADR-013 §10 (the byte-pinned fixture), views-pipeline-core **#280** (the platform pyarrow ceiling), views-faoapi **#348** (the reading half), **#174**. + +--- + +### C-81: What actually gates `main` is weaker than it looks — CI verifies 17 fewer tests than local, and nothing requires it to pass + +| Field | Value | +|-------|-------| +| ID | C-81 | +| Tier | 2 — the guards this arc built to catch cross-repo drift do not run where drift happens, and the branch they protect has no required check. Both halves are structural and both have fired-in-practice evidence. | +| Source | `code-review max` (2026-08-03) — development→main sync audit | +| Trigger | **Coverage half:** when the Appwrite Seam Contract registry next moves — it moved twice on 2026-08-03 alone — nothing in CI will notice; only a maintainer running the suite locally will. **Enforcement half:** the first time someone merges a red PR to `main`. | +| Owner | Simon — both halves need operator action. The coverage half needs a token for two private repositories; the enforcement half is a GitHub console/ruleset change. Neither is engineering work. | +| Location | `.github/workflows/run_pytest.yml`; the `protect_main` ruleset; `tests/conftest.py::sibling_repo` | + +**Coverage.** Measured in an isolated clone, not estimated — **402 collected in every run**, so the whole delta is skips: + +| environment | result | +|---|---| +| local, all siblings present | 362 passed / 40 xfailed / **0 skipped** | +| CI as it was | 347 passed / **17 skipped** / 38 xfailed | +| CI with the views-crafdapi checkout added | 348 passed / **16 skipped** / 38 xfailed | + +*(**This table was wrong twice, and the second time it refuted itself.** Draft one said 361/40 and "same 401" — measured before the same change added a test. Draft two fixed the collected figure to 402 and did not re-derive the rows, so both rows summed to 401 beside an assertion that 402 was collected. The cause of the second error is worth recording: the measurement was taken on a `git clone` of the branch, and a clone carries **committed** state — the new tests were still uncommitted in the working tree. Measuring a claim about your own change requires applying your own change. This is the entry about miscounted tests.)* `sibling_repo` resolves `$VIEWS_` else `../`; in a one-repo checkout neither exists and the tests skip. Skipping is correct behaviour — a missing sibling *is* normal — but the consequence is that **CI verifies strictly less than a developer's laptop, precisely on the assertions that cross a repository boundary.** + +Nine of the seventeen are **new in this arc**, including both registry-drift detectors (pinned edition, commit-reachable-from-`main`) for both partners. Those detectors have a demonstrated drift rate: they fired **twice on 2026-08-03**, hours apart. A detector for a fault that recurs twice in a day, running only on one machine, is most of the way to not existing. + +Where each sibling stands, after trying them: +- **views-crafdapi** — public; its check read source text. Checked out in CI from 2026-08-10, recovering **one** test; **that checkout was removed on 2026-08-12** when the check it served was deleted (ADR-017 §5 erratum, C-92): the cross-seam consumer-document-name pin for CRAF'd. +- **views-datafactory** — public, but its eight tests need the producer's raw GAUL parquets, which are **not in its git repository**. Checking it out converts an honest skip into a `FileNotFoundError`; tried and reverted. +- **views-appwrite** — was private when this was written; **made public 2026-08-08** (`views-appwrite@9d80b75`) and **now checked out in CI**, recovering **seven** tests including both registry-drift detectors. No credential was needed for any of them, and none was ever the obstacle after 2026-08-08 — the obstacle was that this line went on saying "private" for two days after it stopped being true. +- **views-faoapi** — **private**, and the only one. Its single check is dark. Closing it needs either a credential or FAO's consent to make that repository public; the second is being pursued, and ADR-016 §8 carries the trigger for falling back to the first. + +**Enforcement.** `main` is **not branch-protected**: `gh api .../branches/main/protection` returns `404 Branch not protected`, and `gh api .../rules/branches/main` returns `[]`. The `protect_main` ruleset exists and is `active`, but its `ref_name` include-list is **empty**, so it matches nothing — and it declares no `required_status_checks` rule in any case. **A red `Run Pytest` would not block a merge to `main`.** This repository's own `tests/test_falsification_campaign_4_1.py` carries the question as an unverifiable xfail probe; it is verifiable through the API, and the answer is no. + +The two compound: a suite that checks less than you think, and no requirement that even that much passes. Neither is caused by this sync — both are pre-existing — but this sync is the first time `main` receives an epic whose value is largely the guards themselves. + +Cross-refs: **C-46** and **C-57** (both RESOLVED; this is the residual each recorded as *"a CI-cost and cross-repo-coupling decision"* and *"worth deciding once for both"* — it now has a live home and a concrete answer per sibling), **C-80** (the other verification gap found in the same audit), #188. + +**Partial mitigation 2026-08-10 (ADR-016) — the coverage half is mostly closed; the enforcement half is untouched.** + +The coverage half rested on a claim nobody could check. This entry, and the workflow comment it drew on, said `views-appwrite` was **private**, so its seven checks needed a credential. It went public on **2026-08-08** (`views-appwrite@9d80b75`, a deliberate and recorded act), and the claim here went on being made for two days afterwards. No credential was required, and none had been the obstacle since that date. + +CI now checks that repository out and those seven run on every pull request — including both registry-drift detectors, which is what this entry called *"the most valuable of the lot"*. **Measured on the merge run, not derived:** CI went from 16 skips to **9** (`398 passed / 9 skipped / 38 xfailed`, PR #229), the remainder being 8 views-datafactory and 1 views-faoapi. One test also changes character rather than merely un-skipping: the scan refusing registry **values** in this public repository's markdown now runs on the merge rather than only on a maintainer's machine. + +**What remains, and it is two different things:** + +1. **~~One dark check.~~ CLOSED 2026-08-11, and not by a credential.** This read: *"views-faoapi is genuinely private — the consumer-name pin is still laptop-only. That is one test, and it is the one whose failure mode is invisible rather than loud."* ADR-017 moved that check onto the public coordinate registry, so it runs in CI for every partner and needs no access to any private repository. The credential ADR-016 §8 deferred was never issued and is no longer the route. What remains of this half is CRAF'd's consumer-side check (views-crafdapi#53), which is the other repository's work, not a dark check here. +2. **The enforcement half is entirely untouched.** `protect_main`'s ref-name include-list was empty; it now targets the default branch, but **no status check is required**, so a pull request with a red CI can still be merged to `main`. Since merging to `main` *is* the production release, this is the half that matters most and the half that has not moved. + +**And it no longer stands alone — read C-86 before closing this.** Measured 2026-08-10: `protect_main` also lists **zero bypass actors**, so once a status check *is* required, nobody can merge past it, administrator or otherwise. Meanwhile ADR-016 made this repository's CI depend on two other repositories. Adding the required check therefore does two things at once: it closes this entry, and it makes an unbypassable external dependency live on the release path. Both are defensible; doing them in one unremarked step is not. If the escape is wanted, a bypass actor is the same console session. + +**The lesson this entry should carry.** The blocker was not a missing credential. It was a fact about another repository recorded in prose, with no date, that nothing could check — and it survived a console session, an ADR draft and a register entry, all of which repeated it. ADR-016 replaces the prose with a declaration carrying the date it was verified, and a test that fails when CI and the declaration disagree. + +**Update 2026-08-05 — the operator session happened, and neither half of this entry moved.** Simon read the Appwrite console that morning (views-appwrite v1.4.4). It answered two *other* þing-02 questions definitively — there is **no non-production project**, and both platform keys expire 2026-11-17 (now **C-84**) — but the console read is a different action from issuing a token and a different console from GitHub's. So both halves stand: the two private siblings still have no CI credential, and `protect_main`'s ref-name include-list is still empty. + +Recorded rather than left implicit because "the operator did a console session" is exactly the kind of adjacent fact that gets mistaken for progress on this entry. It is not. What it does establish is that the session is a thing that happens, and these two items are small enough to ride along with the next one. + + +--- + +## Disagreements + +### D-12: Post-Run-0 infrastructure & naming intents — repo rename, internal-store transport, compute co-location + +| Field | Value | +|-------|-------| +| ID | D-12 | +| Source | Maintainer direction during the ADR-013 read-through (2026-07-19); assessment in-session | +| Location | Repo-wide (rename); ADR-013 §3/§8 (store transport, co-location); mirrored as dated deferred intents in ADR-013 §8 | + +Three maintainer-raised intents, assessed and **deliberately deferred** — all sequenced strictly after (1) Run 0 proves the wire as adopted and (2) a §3.5 retention owner exists (infrastructure ownership must exist before infrastructure multiplies): + +1. **Rename this repo** to a delivery-screaming name (e.g. `views-delivery`). The repo is already purely delivery code (reconciliation retired to `views_frames_reconcile`, #62 closed 2026-06-26), so the name is the only mismatch with the screaming-architecture rubric. GitHub redirects soften the repo rename; the `views_postprocessing` *package* rename (cross-repo imports, views-models launchers) is the real churn and may trail. +2. **Move `production_forecasts` off Appwrite** to self-managed storage (e.g. Hetzner object storage) — no external consumer reads the internal store. Contract-tolerant: §3 payload + manifest-last semantics are transport-agnostic; only the store-document addressing needs a bounded amendment. +3. **Co-locate delivery compute with the internal store** to kill the ~29 GB/run upload-download round-trip — while **keeping the logical hop** (complete-or-invisible commit marker, hash verification, schedule independence, multi-partner fan-out). Fusing producer and delivery into one machine is **explicitly rejected** — it would rebuild the coupling ADR-013 dissolved. + +**Re-open trigger:** Run 0 verified AND retention owner named — then sequence 2→3 (or 2 alone) as an infrastructure epic, and 1 whenever wire churn is calm. See also C-40 (the migration this rides on), ADR-013 §8. + +**Status 2026-08-03 (both halves re-checked): #131 is CLOSED (2026-07-31), so the verification half of this trigger HAS fired.** The text below was written the day it closed and was already stale; it is corrected rather than deleted because the deferral it holds shut is a live decision. **Run 0 delivered** on 2026-07-27 (first FAO global-land forecast, frame-native, no OOM) and its integrity verification is closed — and #131 also surfaced a liveness dialect gap on the `unfao_delivery` forecast surface. **The retention owner is still unnamed** (ADR-013 §3.5 records the duty as OPEN), and that is now the *only* thing holding this deferral shut. Both halves must hold before it re-opens; one of the two now does. **Naming a retention owner re-opens D-12** — that is an operator decision, not engineering work. Note that intent 2 (move `production_forecasts` off Appwrite) and the unnamed retention owner compound: run-0 added ~110 objects in a single run to a store with no retention policy. --- -### C-72: The pyarrow pin holds this repo inside a high-severity CVE, and the fix changes the wire bytes +### D-11: Pandas→frames seam — concrete siblings + delete vs a polymorphic abstraction | Field | Value | |-------|-------| -| ID | C-72 | -| Tier | 3 — **not currently exploitable here**: the CVE is a *read-path* use-after-free and this repo only **writes** arrow in production (the three `vf_arrow.load` calls are all in tests). It is Tier 3 rather than 4 because the bytes we write are read by views-faoapi under the same platform-wide pin, so the platform's exposure is real even though this repo's is not, and because the remedy collides with a ratified contract. | -| Source | `manual` (2026-08-01) — GitHub surfaced 32 Dependabot alerts on the push that made `main` current; reviewed alert by alert | -| Trigger | When lifting the `pyarrow < 17` ceiling — pipeline-core **#280** (the viewser → views-storage chain) is the platform half — **regenerate the ADR-013 §10 golden fixture in the same change and notify the faoapi seat**, because the upgrade is expected to change the delivered bytes. Also fires if this repo ever gains a production arrow **read**. | -| Location | `pyproject.toml:15` (`pyarrow = ">=16.1.0,<17.0.0"`); `poetry.lock`; the affected operation would be `views_frames.io.arrow.load`, called only at `tests/test_wire_shard.py:90`, `tests/test_wire_fixture.py:73`, `tests/test_wire_header.py:39` | +| ID | D-11 | +| Source | `expert-code-review` (2026-06-28) — review of pandas-migration epic #85 | +| Location | epic #85 / stories #86 (`unfao/frames.py`), #87 (`unfao/extraction.py` + new frame module), #88/#91 (`unfao/managers/unfao.py` source/sink seams) | -**GHSA — "Apache Arrow: potential use-after-free when reading IPC file with pre-buffering."** Vulnerable range `>= 15.0.0, < 23.0.1`; patched in **23.0.1**. This repo's declared pin is `>=16.1.0,<17.0.0` — **entirely inside the vulnerable range, with a ceiling that excludes the fix.** It is the only one of the 32 alerts against a dependency this repo declares itself (`pyproject.toml`); the other 31 are `poetry.lock` resolution — see the disposition note below. +The pandas→views-frames migration (epic #85) deliberately swaps each seam by adding a **concrete** frame-native sibling next to the pandas one and later **deleting** the pandas path — rather than introducing a polymorphic abstraction (an `Extractor` Protocol / a representation port) that both implementations satisfy. -**Why it is not exploitable here, stated precisely rather than reassuringly.** The defect is on the *read* path. Production writes arrow (`contract/wire/shard.py` → `views_frames.io.arrow`) and never reads it back; the only `vf_arrow.load` calls in the repository are the three byte-parity tests above, operating on committed fixtures we generated. So there is no path from partner or producer input to the vulnerable code **in this process**. +- **Position A — concrete siblings + delete (the plan's choice; Martin/Beck-pragmatic, WET-before-DRY).** pandas and frames do **not** coexist at runtime — it is a migration, not a permanent dual representation — so a polymorphic interface would be speculative (YAGNI/ISP: don't force an interface nobody dispatches on). The seam stays readable, each representation is one-concept-per-file, and retirement is a clean file-delete. The invariants already depend on **primitives** (the real abstraction, DIP-satisfied at that boundary), so no port is needed above them. +- **Position B — abstraction/port (Hickey/strict-OCP).** Depending on a representation port would make the swap "extend, not modify," and would let the two paths coexist cleanly during cutover. -**Where the exposure actually sits: views-faoapi.** It reads these shards, under the same platform-wide `pyarrow < 17` ceiling. Our output is its input. This entry does not claim to size that repo's risk — it records that the platform's pyarrow pin is a shared exposure and that the consumer is the reading half. +**Decision: A**, consistent with the maintainer's WET-before-DRY rule and the "migration not coexistence" reality. **Re-open trigger (the one acute case):** S6 (#91) introduces a temporary **dual-write** (legacy parquet + arrow sample-frame) for parity during the faoapi cutover — *if that coexistence proves long-lived* (rather than a brief cutover window), a small abstraction may then earn its place; revisit only then. Until then, concrete-and-delete stands. -**The collision worth understanding — the fix and the contract point in opposite directions.** The five byte-parity failures this repo has treated as "known local toolchain noise" all session are precisely this: the local environment runs **pyarrow 23.0.1 — the patched version** — while CI runs the pinned 16.1.x, and the two produce **different bytes** for the same input. So upgrading to the patched pyarrow is not a dependency bump; it **changes the §10 golden fixture**, which ADR-013 declares normative and which views-faoapi verifies against. Security remediation here is a **contract event**, not maintenance, and must be sequenced with the consumer. +**RE-INSTANTIATED 2026-07-31 (`expert-code-review`) — same tension, new subject; merged here rather than registered as its own disagreement entry.** The review surfaced the identical disagreement one level down, about the **keyed gather** rather than the representation seam. "Given gids, fetch their lookup rows" now exists **three times, all simultaneously live**: a pandas left-merge (`unfao/enrichment.py:117`), an argsort/searchsorted gather (`unfao/historical.py:57-68`), and an isin/filter/sort (`unfao/wire/sidecar.py:47-57`). -**⚠ TWO CORRECTIONS to this entry, made the same day it was written (2026-08-01), on evidence gathered while filing the cross-repo issues.** +- **Position A (GoF, Hickey) — unify now.** Three implementations of one join is a correctness surface, not a migration artifact. They differ in failure semantics: two raise in place naming the absent gids; the pandas one yields NaN and defers to a downstream gate. That divergence is invisible at the call sites. +- **Position B (Feathers, Beck) — defer to #89.** The differing failure semantics are *deliberate*, each implementation is separately tested, and unification means changing three call sites while the legacy branch is still live. The duplication is scheduled to die with that branch. -**(a) views-faoapi is NOT exposed, and the platform ceiling is not uniform.** This entry implied the consumer shared our pin. It does not: faoapi pins **`pyarrow==23.0.1`** — the patched version — in its `pyproject.toml:31`, and it is the repo that actually reads the wire (`forecast/ingestion/wire_reader.py` → `arrow.load`). So nobody on the platform is currently exposed through this path, and the `<17` ceiling views-pipeline-core#280 exists to lift is binding on the **producers** only. One consumer has already moved past it, in production. +**✅ CONDITION MET 2026-08-01 (epic #148).** The legacy forecast branch **was** deleted (#149) — before #89 landed, so the re-open trigger below never fired. Position A is vindicated by events rather than by argument: concrete siblings were built, the migration completed, and the second implementation was deleted rather than abstracted over. A Protocol introduced at decision time would have outlived the thing it existed to unify. Recorded in `tests/test_input_integrity_design_contract.py` ② so the next seam decision has the precedent. -**(b) The re-vendor obligation is stronger than "our fixture changes".** ADR-013 §10 is explicit: *"All three implementing repos' test suites consume the same bytes … The other two repos **vendor a copy** and carry a **pinned root-hash equality test** … A change to the fixture is a change to the contract."* Verified — faoapi's copy is at `tests/forecast/golden/wire_contract/SHA256SUMS`, pinned by `tests/forecast/test_wire_golden_fixture.py`. So the upgrade is a **coordinated three-repo re-vendor**, and it carries an undecided question: whether it bumps `contract_version` (the payload schema does not change, only the encoder's bytes — §10 says a fixture change *is* a contract change). +**Adjudication: B — but conditionally, and the condition is now named.** B is only correct *if the legacy forecast branch is actually deleted*. It had no scheduled deletion PR when this was adjudicated — #149 deleted it on 2026-07-31, which is what made B correct rather than merely preferable — only a "named post-run-0 follow-up" (C-40). If it lingers, three-way duplication becomes the permanent shape and Position A wins retroactively. **Re-open trigger: if the legacy forecast branch still exists when #89 (numpy/pyarrow keyed gather) lands, unify all three consumers onto one fail-loud gather primitive as part of that story rather than deferring again.** Cross-refs C-59 (the duplicate-key hazard that lands differently in each of the three), C-40, #89. -**Filed, so the relocation is complete rather than assumed** (the C-08 lesson): **views-postprocessing#174** (the coordinating issue), **views-faoapi#348** (heads-up + two questions only their seat can answer), and a comment on **views-pipeline-core#280** adding the security dimension and the non-uniform-ceiling finding. +**SETTLED 2026-07-31 (review-rr — the original re-open trigger resolved without firing).** The sole re-open condition was a long-lived S6/#91 dual-write. **#91 is CLOSED**, and the contract path shipped frame-native end-to-end (PRs #115–#129, wire delivered in run-0) without a durable dual-representation window. Position A is vindicated by events: concrete siblings + delete is what actually happened, the pandas path is confined to the legacy branch pending its named post-run-0 deletion, and no polymorphic representation port was ever needed. No live tension remains — this entry is kept as decision provenance for the remaining seam work (#89, #90) rather than as an open question. -Cross-refs: **C-62** (the transitive dependency drag; the other 31 alerts), **C-46** (the datafactory version-state coupling), ADR-013 §10 (the byte-pinned fixture), views-pipeline-core **#280** (the platform pyarrow ceiling), views-faoapi **#348** (the reading half), **#174**. +See also C-40 (the inheritance/representation coupling this migration unwinds), #85 (the migration epic), #45 (the faoapi wire / S6 dual-write). --- -### C-75: `GaulLookupEnricher` has no production caller, and now implements a second copy of the delivery path's keyed gather +### D-09: Multi-store support — parameterize the manager now vs after the FAO global delivery | Field | Value | |-------|-------| -| ID | C-75 | -| Tier | 3 — no correctness impact today: the class is off the delivery path, so a defect in it cannot reach the UN FAO. The cost is that **the verification path and the delivery path now implement the same algorithm twice**, and the tests that check the artifact run through the copy that does *not* ship. A fix applied to one and not the other makes the verification stop verifying what ships — quietly, because both would still pass their own tests. | -| Source | `code-review max` (2026-08-02) — PR #210, five parallel reviewers; two reached this independently | -| Trigger | When a bug is fixed in `contract/historical.py`'s gather (the one that ships), check whether `contract/enrichment.py`'s copy needs the same fix — nothing links them. Also fires at **S5 (#90)**: once the builder is pyarrow-native, the enricher's pandas interface is the last one in the package, and the question "does this class survive?" has to be answered rather than deferred again. | -| Owner | Whoever takes **#90** — the keep-or-retire decision is theirs to make and record, not to defer a third time. Added 2026-08-03: the first draft of this entry named two triggers and no owner, while citing ADR-014 §4 in its own body. This register had already learned that twice — *"a deferral needs an owner and a trigger, not just a reason"* (Cluster L) and *"a decision awaiting an owner, not a task awaiting effort"* (epic #181 closeout). | -| Location | `views_postprocessing/contract/enrichment.py` (the whole class; `_gather` specifically); the shipping twin is `views_postprocessing/contract/historical.py:54-68` | +| ID | D-09 | +| Source | `expert-code-review` (2026-06-12) | +| Perspectives | GoF: the manager is being touched anyway — extract the `DeliveryProfile` now while context is loaded. Beck/Hickey/Ousterhout: the smallest change that delivers wins; a profile refactor adds review surface to the highest-stakes week, and frameworks built under deadline pressure rot. | +| Location | `views_postprocessing/unfao/managers/unfao.py:109-122, 234-247, 272` | +| Status | Open. Review adjudication: **after delivery** — with two exceptions to do now: delete the dead config blocks (`unfao.py:80-107`, a mis-uncomment hazard) and ensure nothing added this week hardcodes additional store identity. The `DeliveryProfile` itself is a calm 1-day job the following week (C-33). | -**Verified, not inferred (2026-08-02):** `grep -rn "GaulLookupEnricher\|enrich_dataframe_with_pg_info"` across the package finds **zero** production callers — the two hits are docstring mentions in `gaul_lookup.py`. The manager calls `gaul_lookup.load()` directly and has zero `enrich` references. **C-66**'s resolution already said this plainly: *"the pandas enricher leaves the delivery path entirely."* +**DEFERRAL EXPIRED 2026-07-31 (review-rr).** The adjudication was "after the FAO global delivery"; **that delivery shipped 2026-07-27** (run-0). Both named exceptions are discharged: the dead config blocks at `unfao.py:80-107` **no longer exist** (verified — that range is now the class definition and `__init__`), and store env **names** were subsequently centralized into `unfao/appwrite_env.py` by þing-01 #134, so nothing added since hardcodes new name literals. What the disagreement actually deferred — the `DeliveryProfile` value object (one manager class, N store configs) — is now unblocked and is the "calm 1-day job." **Not resolved, because the decision it defers is still undecided:** build the profile now, or wait for issue #97's second-store scoping to define what a profile must carry. Recommend deciding that alongside #97; on decision this closes into C-33. -**What PR #210 did, and why that raises the question.** S4 (#89) rewrote this class's lookup side from a pandas merge to a numpy/pyarrow keyed gather: a measured dtype analysis, an empty-lookup guard, a mutation-proven bug fix, a corrected CIC, and five reviewers' attention. All of it spent on a method with no reachable caller outside its own test suite. The engineering is sound; what is missing is anyone having **decided** that the class should exist. +--- -**The duplication is the concrete consequence.** `_gather`'s `argsort → searchsorted → clip → equality-mask` is the same shape as `historical.py:54-68`. The policies differ deliberately — `historical` **raises** on an absent gid (*"geography must never silently vanish"*), the enricher returns nulls for the downstream gate to catch — so extracting a shared helper would mean parameterising the failure policy, which is the guessed abstraction **WET before DRY** exists to prevent. Two copies that are understood is the right call *today*. The trigger above is what stops "today" lasting indefinitely, per **ADR-014 §4**. +## Resolved Concerns -**The precedent is C-45**, `unfao/frames.py`: an unused adapter carried on no live path, resolved by deleting it. This is the same shape with a different module, and the same question — keep it as the declared verification/reference implementation, or retire it and let the fidelity suite test `historical.py` directly. +### C-82: Governance-artifact prose carries numbers and statuses that nothing checks — RESOLVED 2026-08-05 -**Deliberately NOT registered from the same review** (defects in unmerged code, all fixed in #210 before merge rather than tracked): a NaN gid crashing the warning path, the unvalidated int64 coercion at both ends, the AST guard's `else`-branch blind spot, ADR-012's stale pandas-merge claim, and three CIC claims retired elsewhere by #200. The register tracks standing risk; a defect fixed before it ships is not one. They are recorded in the PR. +| Field | Value | +|-------|-------| +| ID | C-82 | +| Tier | 3 — no delivery is affected, but these are the artifacts people plan from. One instance materially under-scopes a planned dependency bump. | +| Source | `code-review max` (2026-08-03) — development→main sync audit | +| Trigger | When the pipeline-core 3.0.0 bump (C-44) is scoped from Cluster M's summary rather than from C-72's body, or when anyone counts on a test-count or issue-state stated in the register. | +| Owner | Whoever runs the next `review-rr` pass; this is curation, not engineering. | +| Location | `reports/technical_risk_register.md` (Clusters I, J, M; D-09, D-11); `docs/CICs/*.md` front matter | -Cross-refs: **C-45** (RESOLVED — the same shape, resolved by deletion), **C-66** (RESOLVED — established the enricher left the delivery path), **C-40** (which calls `enrichment.py` and `extraction.py` together *"the retired-in-place `enrichment.py`/`extraction.py` legacy seams"*), **#89** / **#90** / epic **#85**, ADR-014 §4. +`tests/test_register_integrity.py` checks structure — header counts, section placement, reference resolution — and **no prose at all**. Roughly twenty-five statements drift beneath it. ---- +**The one that would change a decision.** Cluster M declares resolution *"Full for … C-72 …"* at the pipeline-core 3.0.0 bump, while C-72's own body says its fix is gated on pipeline-core **#280** (open), **changes delivered wire bytes**, and requires a coordinated three-repo re-vendor of the ADR-013 §10 golden fixture. Someone planning that bump from the cluster summary under-scopes it badly. Cluster M's heading also says six entries where its body says five. -### C-76: `build_gaul_lookup.py` will write an empty lookup without complaint +**Self-contradiction elsewhere.** Four were named: Cluster I arguing for a `tests/test_register_integrity.py` that already existed with ten green tests; Cluster J naming closed issue **#15** as its fix strategy; D-11 saying a branch *"currently has no scheduled deletion PR"* two paragraphs after recording its deletion; D-09's `Status` row reading *"Open … after delivery"* above prose recording the deferral expired. -| Field | Value | -|-------|-------| -| ID | C-76 | -| Tier | 4 — no silent corruption. A zero-row artifact fails downstream at `historical.build_historical_table`, which raises on cells absent from the lookup. The cost is that it fails **late and confusingly**: the message names missing geography rather than an empty lookup, and the artifact is committed by then. | -| Source | `code-review max` (2026-08-03) — PR #210 second pass, while checking whether the consumer's new guards duplicated a producer guarantee. They do not. | -| Trigger | When `build_gaul_lookup.py` is next run with a new or renamed `--region`, or against a datafactory whose `gaul_admin` parquets have changed shape — check the printed `cells=` count is non-zero before committing the artifact. Nothing else will tell you. | -| Owner | Whoever next runs the builder. It is a two-line guard in a script one person runs by hand, not a scheduling decision. | -| Location | `scripts/build_gaul_lookup.py` — the invariant block at `:246-268` and the write at `:284` | +Three were already corrected by the passes that followed, and only **D-11** survived to this one — measured, not assumed: each was grepped for on 2026-08-05 and Clusters I and J and D-09 returned zero hits. That matters more than the count. The instances got fixed one at a time by whoever tripped over them, which is exactly the failure mode this entry describes: **the register self-heals where someone happens to look and rots everywhere else.** D-11 sat contradicting itself for five days in a section nobody had cause to re-read. -The builder's invariant block is thorough about what it checks: index uniqueness (C-59), nulls in the metadata columns, `-1` sentinels in the code columns (C-35). It does not check that any rows survived. A `--region` argument that filters every cell out, or an upstream join that produces nothing, writes a zero-row parquet and prints `cells=0` as though that were a result. +**Numbers, and this entry's own numbers rotted while it sat open.** It read: *"the count appears as 26 twice and as 18 twice more; the actual is 24."* Since then #90 added a test and the actual became 25, so the entry describing stale counts had a stale count. That is not irony worth savouring — it is the argument. **A count nothing checks is a claim with a half-life**, and the fix is not a more careful re-count. -**Verified 2026-08-03, and the neighbouring worry is NOT real.** The same review asked whether the builder also fails to reject a null key, since `df.isna().sum().sum()` runs *after* `priogrid_gid` becomes the index and `DataFrame.isna()` does not inspect the index. It does not check it — but the null key is unreachable anyway: `df.index.astype("int64")` raises `IntCastingNaNError` two lines earlier. Protection by accident rather than by declaration, which is worth knowing, but not a defect to fix. **Only the empty case is reachable.** +Resolved by removing the volatile numbers rather than correcting them, which is the precedent C-33 was forced into after its measurement was wrong five times: publish the **command**, not the result. `40 permanent guard tests` became the collect-only command; the fidelity count became *"one file discharged three entries"*, which is the claim that mattered and does not move. Historical counts inside dated closure records are left alone — *"18 tests, committed in #141"* was true at #141 and is a record, not a claim about now. -**Why this was found now.** PR #210 added consumer-side refusals for both an empty lookup and a null key to `GaulLookupEnricher.__init__`, and the review challenged them as duplicating a producer guarantee. Checking established the opposite: for the empty case there is no producer guarantee to duplicate, and for the null key the producer's protection is incidental. The consumer guards stay, and this entry records the producer-side half rather than quietly assuming someone will notice. +**CIC front matter.** `GaulLookupEnricher.md` said *Last reviewed 2026-06-18* and `UNFAOPostProcessorManager.md` *2026-06-02* while both bodies carried 2026-08 content — calibrating a reader's trust wrong in the safe direction, which is luck rather than design. The first document was deleted with its class (#90/B3b). The second is corrected, and the field is now checked against the document's own dated content by `test_a_cic_review_date_is_not_older_than_its_own_content` — git could not answer this, because git records when a line was touched and this field claims when someone read the whole thing. -Cross-refs: **C-59** and **C-61** (RESOLVED — the invariant block this sits beside, and the reason it is otherwise thorough), **C-35** (the `-1` defect class it does check for), **C-75** (the consumer whose guards prompted the check), #210. +*The general fix is C-80's, not a re-count:* prose that states a number is a claim, and a claim needs a check. Where a number cannot be checked, the honest move is to state the command that produces it — which is what C-33 was forced into after its measurement was wrong five times. ---- +Cross-refs: **C-80** (the same disease in ADRs and CICs, and the mechanism that would catch both), **C-72** and **C-44** (the bump this mis-scopes), **C-33** (the worked example of publishing the command instead of the result), ADR-014 §1. + +**Resolved 2026-08-05 (B6).** Every named instance is disposed of, and two guards now stand where the prose was unchecked: + +- `test_test_files_named_by_live_entries_exist_or_name_their_repo` — a live entry may not name a test file that does not exist unless it names the repo that owns it. Scoped to Open Concerns and Disagreements on purpose: eleven such mentions exist register-wide and nine are resolved entries correctly recording what discharged them, so a blanket check would cry wolf and be deleted within a day (ADR-014 §3). +- `test_a_cic_review_date_is_not_older_than_its_own_content` — a CIC's `Last reviewed` header may not predate dates in its own body. +**The first draft of the first guard was itself the bug it was written to catch,** and this is the part worth keeping. Its foreign-repo exemption reused `_FOREIGN_PREFIXES` — the list that namespaces *identifiers* — over a sixty-character window. That list holds ordinary English: `models`, `frames`, `pipeline-core`. A mutation planting a vanished file in a live entry left it green, because a sentence three words earlier said *"pinned pipeline-core-free"*. It had found its two real defects by luck of their neighbouring words. Rebuilt to require the owning repo **immediately abutting the path**, and re-proven against three mutations including the one that defeated the draft. A guard nobody has watched fail is decoration (ADR-014 §2) — and a guard watched failing on the *wrong* mutation is worse, because it has a proof attached. +**What is deliberately not fixed.** The general case — arbitrary prose asserting an arbitrary number — is not mechanisable and this entry does not claim it is. What is mechanised is the shape that recurred: a claim naming an artifact, checked by asking whether the artifact exists. Numbers that could not be guarded were removed in favour of the command that produces them, per C-33. + +*Verify:* `pytest -q tests/test_register_integrity.py tests/test_doc_accuracy.py` --- -### C-80: The doc-accuracy scan exempts ADRs and CICs — the two artifact classes that define the contracts +### C-80: The doc-accuracy scan exempts ADRs and CICs — the two artifact classes that define the contracts — RESOLVED | Field | Value | |-------|-------| @@ -492,45 +1145,23 @@ Cross-refs: **C-59** and **C-61** (RESOLVED — the invariant block this sits be Cross-refs: **C-74** (a guard narrower than its declared surface), **C-78** (a guard whose declared scope missed a package), **C-67** (ADR-012 drift, which *is* covered and was caught), ADR-014 §1–§3, #211. ---- - -### C-81: What actually gates `main` is weaker than it looks — CI verifies 17 fewer tests than local, and nothing requires it to pass - -| Field | Value | -|-------|-------| -| ID | C-81 | -| Tier | 2 — the guards this arc built to catch cross-repo drift do not run where drift happens, and the branch they protect has no required check. Both halves are structural and both have fired-in-practice evidence. | -| Source | `code-review max` (2026-08-03) — development→main sync audit | -| Trigger | **Coverage half:** when the Appwrite Seam Contract registry next moves — it moved twice on 2026-08-03 alone — nothing in CI will notice; only a maintainer running the suite locally will. **Enforcement half:** the first time someone merges a red PR to `main`. | -| Owner | Simon — both halves need operator action. The coverage half needs a token for two private repositories; the enforcement half is a GitHub console/ruleset change. Neither is engineering work. | -| Location | `.github/workflows/run_pytest.yml`; the `protect_main` ruleset; `tests/conftest.py::sibling_repo` | - -**Coverage.** Measured in an isolated clone, not estimated — **402 collected in every run**, so the whole delta is skips: - -| environment | result | -|---|---| -| local, all siblings present | 362 passed / 40 xfailed / **0 skipped** | -| CI as it was | 347 passed / **17 skipped** / 38 xfailed | -| CI with the views-crafdapi checkout added | 348 passed / **16 skipped** / 38 xfailed | +**RESOLVED 2026-08-05 (B5).** ADRs and CICs are now scanned, and the design was chosen by measurement rather than by argument. -*(**This table was wrong twice, and the second time it refuted itself.** Draft one said 361/40 and "same 401" — measured before the same change added a test. Draft two fixed the collected figure to 402 and did not re-derive the rows, so both rows summed to 401 beside an assertion that 402 was collected. The cause of the second error is worth recording: the measurement was taken on a `git clone` of the branch, and a clone carries **committed** state — the new tests were still uncommitted in the working tree. Measuring a claim about your own change requires applying your own change. This is the entry about miscounted tests.)* `sibling_repo` resolves `$VIEWS_` else `../`; in a one-repo checkout neither exists and the tests skip. Skipping is correct behaviour — a missing sibling *is* normal — but the consequence is that **CI verifies strictly less than a developer's laptop, precisely on the assertions that cross a repository boundary.** +**What the measurement said.** A path-resolution check over ADRs would have fired **29** times, and inspecting them showed most were correct history, other repositories' files, or paths inside URLs — the cry-wolf outcome this entry predicted, confirmed before building it. The curated deleted-symbol list was the narrower instrument: **22** hits over ADRs, **0** over CICs. Fourteen of the 22 were in ADR-011 alone. -Nine of the seventeen are **new in this arc**, including both registry-drift detectors (pinned edition, commit-reachable-from-`main`) for both partners. Those detectors have a demonstrated drift rate: they fired **twice on 2026-08-03**, hours apart. A detector for a fault that recurs twice in a day, running only on one machine, is most of the way to not existing. +**Three of those hits were real.** ADR-013 still cited `unfao/historical.py`, `unfao/wire/` and `unfao/wire/source_selection.py` — all moved to `contract/` by #153, all fixed here. Three genuine defects hiding among five markable ones is a workable ratio, and it is the ratio that justified turning the scan on. -Where each sibling stands, after trying them: -- **views-crafdapi** — public, its check reads source text. **Now checked out in CI**, recovering **one** test: the cross-seam consumer-document-name pin for CRAF'd. -- **views-datafactory** — public, but its eight tests need the producer's raw GAUL parquets, which are **not in its git repository**. Checking it out converts an honest skip into a `FileNotFoundError`; tried and reverted. -- **views-appwrite**, **views-faoapi** — **private**. The most valuable checks live here. Closing this needs a token in CI. +**Two escapes, both declared rather than inferred.** Line-scoped `legacy-ok` for an isolated historical mention; a new file-level `` for a document whose *subject* is a retirement. ADR-011 is the case that earned it — it **is** the decision to remove the runtime mapper, so its subject appears fourteen times, correctly. A second guard pins the set of file-level exemptions to that one document, so adding another shows up in a diff. -**Enforcement.** `main` is **not branch-protected**: `gh api .../branches/main/protection` returns `404 Branch not protected`, and `gh api .../rules/branches/main` returns `[]`. The `protect_main` ruleset exists and is `active`, but its `ref_name` include-list is **empty**, so it matches nothing — and it declares no `required_status_checks` rule in any case. **A red `Run Pytest` would not block a merge to `main`.** This repository's own `tests/test_falsification_campaign_4_1.py` carries the question as an unverifiable xfail probe; it is verifiable through the API, and the answer is no. +**And a check nothing else could have made.** `test_a_cic_does_not_name_a_collaborator_its_class_never_calls` asserts that a class a CIC names is actually referenced by the class it documents — the `GaulLookupEnricher` failure, which no path check and no symbol list would have caught at the time, because the class existed and the paths resolved. Exception types are excluded: a *collaborator* is something the class reaches for, an *exception* something that passes through, and the first draft flagged three exceptions the manager legitimately propagates. -The two compound: a suite that checks less than you think, and no requirement that even that much passes. Neither is caused by this sync — both are pre-existing — but this sync is the first time `main` receives an epic whose value is largely the guards themselves. +**The mutation campaign found a miss in the previous change.** Reintroducing the exact `GaulLookupEnricher` sentence did **not** fail — the collaborator check only sees classes that still exist, and #90/C-75 had deleted that one. The real gap was that the deletion never extended the deleted-symbol list, which that list's own comment demands in as many words: *"A deletion PR that does not extend this regex has not finished."* Extended here; the reintroduction now fails. Four further historical mentions written yesterday were flagged by the extension and marked. -Cross-refs: **C-46** and **C-57** (both RESOLVED; this is the residual each recorded as *"a CI-cost and cross-repo-coupling decision"* and *"worth deciding once for both"* — it now has a live home and a concrete answer per sibling), **C-80** (the other verification gap found in the same audit), #188. +Cross-refs: **C-75** (the retirement whose CIC error motivated this, and whose PR the mutation test caught short), **C-74** and **C-78** (guards narrower than their declared surface), **C-82** (the register's own prose, still unscanned), ADR-014 §2 and §3. --- -### C-83: A queryset that fails to import is reported as a queryset that declares the wrong format +### C-83: A queryset that fails to import is reported as a queryset that declares the wrong format — RESOLVED | Field | Value | |-------|-------| @@ -555,36 +1186,17 @@ This is ADR-003's rule broken by composition rather than by anyone inferring any Cross-refs: **C-44** (the bump whose review found this), **C-40** (the inherited surface it arrives through), Cluster J (the *no* vs *could not tell* family), ADR-003, #126, #149. ---- - -### C-82: Governance-artifact prose carries numbers and statuses that nothing checks - -| Field | Value | -|-------|-------| -| ID | C-82 | -| Tier | 3 — no delivery is affected, but these are the artifacts people plan from. One instance materially under-scopes a planned dependency bump. | -| Source | `code-review max` (2026-08-03) — development→main sync audit | -| Trigger | When the pipeline-core 3.0.0 bump (C-44) is scoped from Cluster M's summary rather than from C-72's body, or when anyone counts on a test-count or issue-state stated in the register. | -| Owner | Whoever runs the next `review-rr` pass; this is curation, not engineering. | -| Location | `reports/technical_risk_register.md` (Clusters I, J, M; D-09, D-11); `docs/CICs/*.md` front matter | - -`tests/test_register_integrity.py` checks structure — header counts, section placement, reference resolution — and **no prose at all**. Roughly twenty-five statements drift beneath it. - -**The one that would change a decision.** Cluster M declares resolution *"Full for … C-72 …"* at the pipeline-core 3.0.0 bump, while C-72's own body says its fix is gated on pipeline-core **#280** (open), **changes delivered wire bytes**, and requires a coordinated three-repo re-vendor of the ADR-013 §10 golden fixture. Someone planning that bump from the cluster summary under-scopes it badly. Cluster M's heading also says six entries where its body says five. - -**Self-contradiction elsewhere.** Cluster I still argues that *"there is no equivalent for the register — a small `tests/test_register_integrity.py` … would make this class self-detecting"*; that file exists, has ten green tests, and is cited elsewhere in the same document. Cluster J names issue **#15** as its fix strategy; #15 is closed and superseded by `docs/operations/correction_procedure.md`. D-11 says a branch *"currently has no scheduled deletion PR"* two paragraphs after recording that it was deleted. D-09's `Status` row reads *"Open … after delivery"* directly above prose recording the deferral expired on 2026-07-31. +**RESOLVED 2026-08-05 (B4).** `launch_config.assert_queryset_was_importable(queryset)` now runs **before** the format check, and both managers read the queryset once and reuse the value. -**Numbers.** The `test_gaul_lookup_fidelity.py` count appears as **26** twice in the register and as **18** twice more including `test_register_integrity.py`'s own docstring; the actual is **24**, and 26 was never true — it was written when the file held 24. Also *"40 ADR-013 guard tests"* (39) and *"`test_enrichment.py`, 16"* (39). +The refusal says what actually happened — *"the postprocessor's config_queryset could not be imported … This is NOT a declaration problem: do not edit data_format until the module imports"* — and steers the operator toward the traceback pipeline-core logged, and toward a missing sibling checkout or dependency. It logs before it raises (ADR-008). -**CIC front matter.** `GaulLookupEnricher.md` says *Last reviewed 2026-06-18* and `UNFAOPostProcessorManager.md` *2026-06-02*, while both bodies carry 2026-08 content. A reader calibrating trust from the header calibrates it wrong in the safe direction, which is lucky rather than designed. +**Three guards, because order is the fix.** One proves the refusal fires and names the real fault; one proves an importable queryset passes (the format question belongs to the *next* check, and keeping them separate is the whole point); one asserts, per partner, that `get_queryset()` is called exactly once and that importability is checked first. Mutation-proven by deleting the check and by reversing the order — both fail. -*The general fix is C-80's, not a re-count:* prose that states a number is a claim, and a claim needs a check. Where a number cannot be checked, the honest move is to state the command that produces it — which is what C-33 was forced into after its measurement was wrong five times. - -Cross-refs: **C-80** (the same disease in ADRs and CICs, and the mechanism that would catch both), **C-72** and **C-44** (the bump this mis-scopes), **C-33** (the worked example of publishing the command instead of the result), ADR-014 §1. +**What is not fixed here, deliberately.** Upstream still returns `None` for any import exception, so the ambiguity exists at its source; we simply stopped passing it into a function whose contract is to default. Raising upstream would be better and is not ours to do — and waiting for it would have left the misleading message on the live FAO path meanwhile. --- -### C-79: `_ContractStorePort.upload`'s result check is called "the whole mechanism" and has no test, and it fails open +### C-79: `_ContractStorePort.upload`'s result check is called "the whole mechanism" and has no test, and it fails open — RESOLVED | Field | Value | |-------|-------| @@ -607,102 +1219,141 @@ Neither is urgent, because `OperationResult.success` is typed `bool` and is neve Cross-refs: **C-40** (the pipeline-core surface this port wraps), **C-44** (the 3.0.0 bump that is the named trigger), **C-77** (the other unguarded thing on the same delivery leg), ADR-014 §1, #211, #146. +**RESOLVED 2026-08-05 (B4).** Two changes, and the second is the one that mattered. + +**Polarity.** `if success is False` became `if success is not True`. The old form failed **open**: a result that was `None`, or lacked the attribute, or carried a non-bool, sailed through as though the upload had worked. The dead `to_dict()` fallback went with it — an unrecognised result should be refused and *named*, not adapted to silently. The refusal now reports what it actually received, because `success=None` (a moved contract) and `success=False` (a reported failure) are different faults and send an operator to different places. + +**Tests, where there were none.** `tests/test_store_port.py` — 16 tests over both partners: the happy path, a reported failure carrying the store's own error, four unrecognised-result shapes, and the field list the port forwards. Mutation-proven by reverting the polarity, which fails four of them. + +**The standing excuse never applied here.** Manager-side facts are source-scanned because the managers need Appwrite env and a views-models path manager to instantiate. `_ContractStorePort` needs neither — it takes a store object and calls four methods on it. A fake store was always enough; nobody had tried. + +**Its trigger fired two days before this and nobody noticed.** The entry read *"the 3.0.0 bump is the next occasion"*; the bump landed 2026-08-03, C-44 was closed with a wheel-level verification of the suite, and the return contract was never re-read. `test_register_integrity.py` cannot catch that — its checks are structural and none evaluates whether a named external event has occurred. That gap is C-82's. + --- -### C-77: The historical leg names its document from the model path, not from the declared consumer name — and nothing checks the two agree +### C-75: `GaulLookupEnricher` has no production caller, and now implements a second copy of the delivery path's keyed gather — RESOLVED | Field | Value | |-------|-------| -| ID | C-77 | -| Tier | 2 — structural fragility with a clear trigger, affecting **both** partners. Not Tier 1: the failure is a document the consumer cannot find, not a wrong value inside one. But it is the **F1 invisibility shape** — ADR-013 §4.1a, the defect that left six `orange_ensemble` forecast documents stranded in `unfao_bucket` while forecast serving read empty for months. Nobody notices a delivery that simply is not there. | -| Source | `code-review max` (2026-08-03) — PR #211, cross-checking the crafd producer against the views-crafdapi consumer | -| Trigger | When a postprocessor's directory is renamed in views-models, or a new partner package is added whose directory name differs from its `CONSUMER_DOCUMENT_NAME` — check that the historical artifact is still retrievable by the consumer's filter. The forecast leg will keep working, so a green delivery run is not evidence. | -| Owner | Whoever takes the guard. It is a one-line assertion plus a test, not a design decision — but it must be taken deliberately, because the current agreement is a coincidence nobody has written down. | -| Location | The historical-artifact upload in `views_postprocessing//managers/.py` — the call passing `name=self._model_path.model_name`, in `_save_contract`. For contrast, the correct leg is the `consumer_name=product.CONSUMER_DOCUMENT_NAME` argument a few lines above, which reaches the wire as `common["name"]` in `contract/wire/sink.py::deliver_run`. | +| ID | C-75 | +| Tier | 3 — no correctness impact today: the class is off the delivery path, so a defect in it cannot reach the UN FAO. The cost is that **the verification path and the delivery path now implement the same algorithm twice**, and the tests that check the artifact run through the copy that does *not* ship. A fix applied to one and not the other makes the verification stop verifying what ships — quietly, because both would still pass their own tests. | +| Source | `code-review max` (2026-08-02) — PR #210, five parallel reviewers; two reached this independently | +| Trigger | When a bug is fixed in `contract/historical.py`'s gather (the one that ships), check whether `contract/enrichment.py`'s copy needs the same fix — nothing links them. Also fires at **S5 (#90)**: once the builder is pyarrow-native, the enricher's pandas interface is the last one in the package, and the question "does this class survive?" has to be answered rather than deferred again. | +| Owner | Whoever takes **#90** — the keep-or-retire decision is theirs to make and record, not to defer a third time. Added 2026-08-03: the first draft of this entry named two triggers and no owner, while citing ADR-014 §4 in its own body. This register had already learned that twice — *"a deferral needs an owner and a trigger, not just a reason"* (Cluster L) and *"a decision awaiting an owner, not a task awaiting effort"* (epic #181 closeout). | +| Location | `views_postprocessing/contract/enrichment.py` (the whole class; `_gather` specifically); the shipping twin is `views_postprocessing/contract/historical.py:54-68` | -The forecast leg is right. It threads the declared constant through: the manager passes `consumer_name=product.CONSUMER_DOCUMENT_NAME` into `deliver_run`, which sets `common = {"name": consumer_name, ...}`. One declaration, carried to the wire as a parameter — the shape C-69 credited as already correct. +**Verified, not inferred (2026-08-02):** `grep -rn "GaulLookupEnricher\|enrich_dataframe_with_pg_info"` across the package finds **zero** production callers — the two hits are docstring mentions in `gaul_lookup.py`. The manager calls `gaul_lookup.load()` directly and has zero `enrich` references. **C-66**'s resolution already said this plainly: *"the pandas enricher leaves the delivery path entirely."* -**The historical-actuals leg does not use that constant at all.** It passes `name=self._model_path.model_name` — a value that comes from the postprocessor's *directory name* in views-models, not from any declaration in this repository. The consumer filters on exactly the string this repo declares: `filters["name"] = self.model_path.model_name`, where the path manager is constructed as `APIPathManager("un_crafd")`. +**What PR #210 did, and why that raises the question.** S4 (#89) rewrote this class's lookup side from a pandas merge to a numpy/pyarrow keyed gather: a measured dtype analysis, an empty-lookup guard, a mutation-proven bug fix, a corrected CIC, and five reviewers' attention. All of it spent on a method with no reachable caller outside its own test suite. The engineering is sound; what is missing is anyone having **decided** that the class should exist. -**For FAO the two agree; for CRAF'd nobody can yet say.** `views-models/postprocessors/` contains `un_fao` and nothing else — there is **no `un_crafd` postprocessor directory**, so CRAF'd's historical `name=` has never been resolved, let alone compared against its consumer's filter. That makes this worse rather than better: for the live partner the agreement is a coincidence nobody wrote down, and for the new one it is an assumption that will first be tested by a production run. Whoever creates that directory decides, without knowing it, whether CRAF'd's actuals are retrievable. +**The duplication is the concrete consequence.** `_gather`'s `argsort → searchsorted → clip → equality-mask` is the same shape as `historical.py:54-68`. The policies differ deliberately — `historical` **raises** on an absent gid (*"geography must never silently vanish"*), the enricher returns nulls for the downstream gate to catch — so extracting a shared helper would mean parameterising the failure policy, which is the guessed abstraction **WET before DRY** exists to prevent. Two copies that are understood is the right call *today*. The trigger above is what stops "today" lasting indefinitely, per **ADR-014 §4**. -**Nothing in this repository asserts they agree.** `tests/test_product.py` asserts `CONSUMER_DOCUMENT_NAME` for the forecast leg; `tests/test_hop_b_sink_e2e.py` checks `consumer_name` on the forecast leg. Neither touches the historical leg's `name=`. A rename of the views-models directory — an ordinary, plausible act, done in a different repository by someone who has never read this file — silently detaches the historical artifact from the consumer's filter while every test here stays green and every delivery run reports success. +**The precedent is C-45**, `unfao/frames.py`: an unused adapter carried on no live path, resolved by deleting it. This is the same shape with a different module, and the same question — keep it as the declared verification/reference implementation, or retire it and let the fidelity suite test `historical.py` directly. -This is ADR-003's rule broken in the quiet direction: the delivery **infers** its consumer identity from a path instead of reading the declaration that exists three lines away. It is also the fourth home for partner identity, where C-69's 2026-07-31 note counted three and recommended consolidation rather than relocation. Consolidation did not reach this line. +**DECISION 2026-08-04 (#90), which this entry's Owner field required of whoever took it: RETIRE.** -**Scope note:** the crafd package inherited this unchanged from `unfao`; PR #211 did not introduce it, it doubled it. Registering it against both partners rather than against the PR. +The conditions are no longer arguable. The class has **zero production callers** — only three test files import it. Its last stated justification was "the build/verification path", and #90 rewrote that path arrow-native without touching it, so the justification is spent. It holds the package's **last pandas reference** (a `TYPE_CHECKING` import), which is the one thing standing between epic #85 and an honest close. And its `_gather` duplicates `contract/historical.py`'s shipping gather, which is independently covered by four test files. -Cross-refs: **C-01** (RESOLVED — the metadata-completeness gate; same partner, same delivery, different field), **C-69** (RESOLVED — "partner identity has THREE homes"; this is the fourth and the note's consolidation recommendation is the fix), **C-33** (the duplication that turned one instance into two), ADR-013 §4.1a (F1 invisibility), ADR-003 (declarations over inference), #211. +**C-45 is the precedent and it was resolved by deleting.** Same shape, different module. ---- +**Not executed in #90, deliberately.** The retirement touches ten files — the module, its 39 tests, references in two other test files, its CIC, ADR-012, `gaul_lookup.py`'s docstring, the machinery list in `test_clone_readiness.py`, and the pandas-importer assertion in `test_doc_accuracy.py`. Folding that into a builder rewrite would mix a behaviour-preserving change with a large deletion, which is the thing epic #148's S5 explicitly refused to do. It is the next change, not a later one. -## Disagreements +**Epic #85 and tracking #93 stay open until it lands**, because their claim — pandas pushed to the seams — only becomes true when this module is gone. -### D-12: Post-Run-0 infrastructure & naming intents — repo rename, internal-store transport, compute co-location +**Deliberately NOT registered from the same review** (defects in unmerged code, all fixed in #210 before merge rather than tracked): a NaN gid crashing the warning path, the unvalidated int64 coercion at both ends, the AST guard's `else`-branch blind spot, ADR-012's stale pandas-merge claim, and three CIC claims retired elsewhere by #200. The register tracks standing risk; a defect fixed before it ships is not one. They are recorded in the PR. -| Field | Value | -|-------|-------| -| ID | D-12 | -| Source | Maintainer direction during the ADR-013 read-through (2026-07-19); assessment in-session | -| Location | Repo-wide (rename); ADR-013 §3/§8 (store transport, co-location); mirrored as dated deferred intents in ADR-013 §8 | +Cross-refs: **C-45** (RESOLVED — the same shape, resolved by deletion), **C-66** (RESOLVED — established the enricher left the delivery path), **C-40** (which calls `enrichment.py` and `extraction.py` together *"the retired-in-place `enrichment.py`/`extraction.py` legacy seams"*), **#89** / **#90** / epic **#85**, ADR-014 §4. -Three maintainer-raised intents, assessed and **deliberately deferred** — all sequenced strictly after (1) Run 0 proves the wire as adopted and (2) a §3.5 retention owner exists (infrastructure ownership must exist before infrastructure multiplies): +**RESOLVED 2026-08-04 — retired, as the decision recorded above required.** `views_postprocessing/contract/enrichment.py` and its 39 tests are deleted, along with `docs/CICs/GaulLookupEnricher.md`. -1. **Rename this repo** to a delivery-screaming name (e.g. `views-delivery`). The repo is already purely delivery code (reconciliation retired to `views_frames_reconcile`, #62 closed 2026-06-26), so the name is the only mismatch with the screaming-architecture rubric. GitHub redirects soften the repo rename; the `views_postprocessing` *package* rename (cross-repo imports, views-models launchers) is the real churn and may trail. -2. **Move `production_forecasts` off Appwrite** to self-managed storage (e.g. Hetzner object storage) — no external consumer reads the internal store. Contract-tolerant: §3 payload + manifest-last semantics are transport-agnostic; only the store-document addressing needs a bounded amendment. -3. **Co-locate delivery compute with the internal store** to kill the ~29 GB/run upload-download round-trip — while **keeping the logical hop** (complete-or-invisible commit marker, hash verification, schedule independence, multi-partner fan-out). Fusing producer and delivery into one machine is **explicitly rejected** — it would rebuild the coupling ADR-013 dissolved. +**No coverage of shipping code was lost.** The two tests elsewhere that imported the class both asserted only that its `lookup_version` agreed with `gaul_lookup.version()` — two readers of one fact, checked against each other. They now read the fact through the declared reader the delivery itself uses, which is the half that was ever load-bearing. The gather it duplicated is `contract/historical.py`'s, covered independently by four test files. -**Re-open trigger:** Run 0 verified AND retention owner named — then sequence 2→3 (or 2 alone) as an infrastructure epic, and 1 whenever wire churn is calm. See also C-40 (the migration this rides on), ADR-013 §8. +**One guard was deleted rather than kept.** `test_gaul_lookup_access.py` asserted `"GaulLookupEnricher" not in` the manager source. With the class gone that assertion cannot fail, and a test that cannot fail is decoration (ADR-014 §2). What it protected — one lookup read per delivery — is the first assertion in the same function and still bites. -**Status 2026-08-03 (both halves re-checked): #131 is CLOSED (2026-07-31), so the verification half of this trigger HAS fired.** The text below was written the day it closed and was already stale; it is corrected rather than deleted because the deferral it holds shut is a live decision. **Run 0 delivered** on 2026-07-27 (first FAO global-land forecast, frame-native, no OOM) and its integrity verification is closed — and #131 also surfaced a liveness dialect gap on the `unfao_delivery` forecast surface. **The retention owner is still unnamed** (ADR-013 §3.5 records the duty as OPEN), and that is now the *only* thing holding this deferral shut. Both halves must hold before it re-opens; one of the two now does. **Naming a retention owner re-opens D-12** — that is an operator decision, not engineering work. Note that intent 2 (move `production_forecasts` off Appwrite) and the unnamed retention owner compound: run-0 added ~110 objects in a single run to a store with no retention policy. +**Fifteen files, and the sweep is the point.** The module, its tests, its CIC, the CIC index, two test files that imported it, the machinery list, the pandas-importer assertion, `README.md`'s dependency table and package tree, `role_and_seams.md`'s tree and contract list, ADR-012's ontology row and its pandas claim, and two module docstrings. Every one of those was a live claim about a class that no longer exists — which is the argument for C-80: none of the ADR or CIC references would have been caught by any guard. + +**What it makes true.** `grep -rn "^import pandas\|^from pandas" views_postprocessing/ scripts/` now returns nothing at all — not a runtime import, not a type-only one. Epic **#85**'s claim, *pandas pushed to the seams*, is finally literal rather than nearly-true, and #85 and #93 close with this. --- -### D-11: Pandas→frames seam — concrete siblings + delete vs a polymorphic abstraction +### C-76: `build_gaul_lookup.py` will write an empty lookup without complaint — RESOLVED | Field | Value | |-------|-------| -| ID | D-11 | -| Source | `expert-code-review` (2026-06-28) — review of pandas-migration epic #85 | -| Location | epic #85 / stories #86 (`unfao/frames.py`), #87 (`unfao/extraction.py` + new frame module), #88/#91 (`unfao/managers/unfao.py` source/sink seams) | +| ID | C-76 | +| Tier | 4 — no silent corruption. A zero-row artifact fails downstream at `historical.build_historical_table`, which raises on cells absent from the lookup. The cost is that it fails **late and confusingly**: the message names missing geography rather than an empty lookup, and the artifact is committed by then. | +| Source | `code-review max` (2026-08-03) — PR #210 second pass, while checking whether the consumer's new guards duplicated a producer guarantee. They do not. | +| Trigger | When `build_gaul_lookup.py` is next run with a new or renamed `--region`, or against a datafactory whose `gaul_admin` parquets have changed shape — check the printed `cells=` count is non-zero before committing the artifact. Nothing else will tell you. | +| Owner | Whoever next runs the builder. It is a two-line guard in a script one person runs by hand, not a scheduling decision. | +| Location | `scripts/build_gaul_lookup.py` — the invariant block at `:246-268` and the write at `:284` | -The pandas→views-frames migration (epic #85) deliberately swaps each seam by adding a **concrete** frame-native sibling next to the pandas one and later **deleting** the pandas path — rather than introducing a polymorphic abstraction (an `Extractor` Protocol / a representation port) that both implementations satisfy. +The builder's invariant block is thorough about what it checks: index uniqueness (C-59), nulls in the metadata columns, `-1` sentinels in the code columns (C-35). It does not check that any rows survived. A `--region` argument that filters every cell out, or an upstream join that produces nothing, writes a zero-row parquet and prints `cells=0` as though that were a result. -- **Position A — concrete siblings + delete (the plan's choice; Martin/Beck-pragmatic, WET-before-DRY).** pandas and frames do **not** coexist at runtime — it is a migration, not a permanent dual representation — so a polymorphic interface would be speculative (YAGNI/ISP: don't force an interface nobody dispatches on). The seam stays readable, each representation is one-concept-per-file, and retirement is a clean file-delete. The invariants already depend on **primitives** (the real abstraction, DIP-satisfied at that boundary), so no port is needed above them. -- **Position B — abstraction/port (Hickey/strict-OCP).** Depending on a representation port would make the swap "extend, not modify," and would let the two paths coexist cleanly during cutover. +**Verified 2026-08-03, and the neighbouring worry is NOT real.** The same review asked whether the builder also fails to reject a null key, since `df.isna().sum().sum()` runs *after* `priogrid_gid` becomes the index and `DataFrame.isna()` does not inspect the index. It does not check it — but the null key is unreachable anyway: `df.index.astype("int64")` raises `IntCastingNaNError` two lines earlier. Protection by accident rather than by declaration, which is worth knowing, but not a defect to fix. **Only the empty case is reachable.** -**Decision: A**, consistent with the maintainer's WET-before-DRY rule and the "migration not coexistence" reality. **Re-open trigger (the one acute case):** S6 (#91) introduces a temporary **dual-write** (legacy parquet + arrow sample-frame) for parity during the faoapi cutover — *if that coexistence proves long-lived* (rather than a brief cutover window), a small abstraction may then earn its place; revisit only then. Until then, concrete-and-delete stands. +**Why this was found now.** PR #210 added consumer-side refusals for both an empty lookup and a null key to `GaulLookupEnricher.__init__`, and the review challenged them as duplicating a producer guarantee. Checking established the opposite: for the empty case there is no producer guarantee to duplicate, and for the null key the producer's protection is incidental. The consumer guards stay, and this entry records the producer-side half rather than quietly assuming someone will notice. -**RE-INSTANTIATED 2026-07-31 (`expert-code-review`) — same tension, new subject; merged here rather than registered as its own disagreement entry.** The review surfaced the identical disagreement one level down, about the **keyed gather** rather than the representation seam. "Given gids, fetch their lookup rows" now exists **three times, all simultaneously live**: a pandas left-merge (`unfao/enrichment.py:117`), an argsort/searchsorted gather (`unfao/historical.py:57-68`), and an isin/filter/sort (`unfao/wire/sidecar.py:47-57`). +Cross-refs: **C-59** and **C-61** (RESOLVED — the invariant block this sits beside, and the reason it is otherwise thorough), **C-35** (the `-1` defect class it does check for), **C-75** (the consumer whose guards prompted the check), #210. -- **Position A (GoF, Hickey) — unify now.** Three implementations of one join is a correctness surface, not a migration artifact. They differ in failure semantics: two raise in place naming the absent gids; the pandas one yields NaN and defers to a downstream gate. That divergence is invisible at the call sites. -- **Position B (Feathers, Beck) — defer to #89.** The differing failure semantics are *deliberate*, each implementation is separately tested, and unification means changing three call sites while the legacy branch is still live. The duplication is scheduled to die with that branch. +--- -**✅ CONDITION MET 2026-08-01 (epic #148).** The legacy forecast branch **was** deleted (#149) — before #89 landed, so the re-open trigger below never fired. Position A is vindicated by events rather than by argument: concrete siblings were built, the migration completed, and the second implementation was deleted rather than abstracted over. A Protocol introduced at decision time would have outlived the thing it existed to unify. Recorded in `tests/test_input_integrity_design_contract.py` ② so the next seam decision has the precedent. +**RESOLVED 2026-08-04 (#90).** `build()` now refuses a zero-row result: -**Adjudication: B — but conditionally, and the condition is now named.** B is only correct *if the legacy forecast branch is actually deleted*. It currently has no scheduled deletion PR — only a "named post-run-0 follow-up" (C-40). If it lingers, three-way duplication becomes the permanent shape and Position A wins retroactively. **Re-open trigger: if the legacy forecast branch still exists when #89 (numpy/pyarrow keyed gather) lands, unify all three consumers onto one fail-loud gather primitive as part of that story rather than deferring again.** Cross-refs C-59 (the duplicate-key hazard that lands differently in each of the three), C-40, #89. +> the build produced ZERO cells for region 'land_gaul'. Either the region filtered every cell out, or the join found no overlap between the seven source parquets. An empty lookup is writable and looks like a result; it is not one. -**SETTLED 2026-07-31 (review-rr — the original re-open trigger resolved without firing).** The sole re-open condition was a long-lived S6/#91 dual-write. **#91 is CLOSED**, and the contract path shipped frame-native end-to-end (PRs #115–#129, wire delivered in run-0) without a durable dual-representation window. Position A is vindicated by events: concrete siblings + delete is what actually happened, the pandas path is confined to the legacy branch pending its named post-run-0 deletion, and no polymorphic representation port was ever needed. No live tension remains — this entry is kept as decision provenance for the remaining seam work (#89, #90) rather than as an open question. +Pinned by `tests/test_gaul_lookup_fidelity.py::test_builder_refuses_a_build_with_zero_cells`. -See also C-40 (the inheritance/representation coupling this migration unwinds), #85 (the migration epic), #45 (the faoapi wire / S6 dual-write). +**The guard did not survive its own first test, and that is worth recording.** On an empty table `pa.array([True] * 0)` infers NULL type, so `pc.and_` in the completeness filter raised `ArrowNotImplementedError` *before* the zero-row check could speak — the confusing-late-failure this entry exists to prevent, relocated by one function. The mask is now explicitly `pa.bool_()`. A guard written and not watched fail is decoration (ADR-014 §2); this one was watched, failed for the wrong reason, and was fixed. --- -### D-09: Multi-store support — parameterize the manager now vs after the FAO global delivery +### C-77: The historical leg names its document from the model path, not from the declared consumer name — and nothing checks the two agree — RESOLVED | Field | Value | |-------|-------| -| ID | D-09 | -| Source | `expert-code-review` (2026-06-12) | -| Perspectives | GoF: the manager is being touched anyway — extract the `DeliveryProfile` now while context is loaded. Beck/Hickey/Ousterhout: the smallest change that delivers wins; a profile refactor adds review surface to the highest-stakes week, and frameworks built under deadline pressure rot. | -| Location | `views_postprocessing/unfao/managers/unfao.py:109-122, 234-247, 272` | -| Status | Open. Review adjudication: **after delivery** — with two exceptions to do now: delete the dead config blocks (`unfao.py:80-107`, a mis-uncomment hazard) and ensure nothing added this week hardcodes additional store identity. The `DeliveryProfile` itself is a calm 1-day job the following week (C-33). | +| ID | C-77 | +| Tier | 2 — structural fragility with a clear trigger, affecting **both** partners. Not Tier 1: the failure is a document the consumer cannot find, not a wrong value inside one. But it is the **F1 invisibility shape** — ADR-013 §4.1a, the defect that left six `orange_ensemble` forecast documents stranded in `unfao_bucket` while forecast serving read empty for months. Nobody notices a delivery that simply is not there. | +| Source | `code-review max` (2026-08-03) — PR #211, cross-checking the crafd producer against the views-crafdapi consumer | +| Trigger | When a postprocessor's directory is renamed in views-models, or a new partner package is added whose directory name differs from its `CONSUMER_DOCUMENT_NAME` — check that the historical artifact is still retrievable by the consumer's filter. The forecast leg will keep working, so a green delivery run is not evidence. | +| Owner | Whoever takes the guard. It is a one-line assertion plus a test, not a design decision — but it must be taken deliberately, because the current agreement is a coincidence nobody has written down. | +| Location | The historical-artifact upload in `views_postprocessing//managers/.py` — the call passing `name=self._model_path.model_name`, in `_save_contract`. For contrast, the correct leg is the `consumer_name=product.CONSUMER_DOCUMENT_NAME` argument a few lines above, which reaches the wire as `common["name"]` in `contract/wire/sink.py::deliver_run`. | -**DEFERRAL EXPIRED 2026-07-31 (review-rr).** The adjudication was "after the FAO global delivery"; **that delivery shipped 2026-07-27** (run-0). Both named exceptions are discharged: the dead config blocks at `unfao.py:80-107` **no longer exist** (verified — that range is now the class definition and `__init__`), and store env **names** were subsequently centralized into `unfao/appwrite_env.py` by þing-01 #134, so nothing added since hardcodes new name literals. What the disagreement actually deferred — the `DeliveryProfile` value object (one manager class, N store configs) — is now unblocked and is the "calm 1-day job." **Not resolved, because the decision it defers is still undecided:** build the profile now, or wait for issue #97's second-store scoping to define what a profile must carry. Recommend deciding that alongside #97; on decision this closes into C-33. +The forecast leg is right. It threads the declared constant through: the manager passes `consumer_name=product.CONSUMER_DOCUMENT_NAME` into `deliver_run`, which sets `common = {"name": consumer_name, ...}`. One declaration, carried to the wire as a parameter — the shape C-69 credited as already correct. ---- +**The historical-actuals leg does not use that constant at all.** It passes `name=self._model_path.model_name` — a value that comes from the postprocessor's *directory name* in views-models, not from any declaration in this repository. The consumer filters on exactly the string this repo declares: `filters["name"] = self.model_path.model_name`, where the path manager is constructed as `APIPathManager("un_crafd")`. -## Resolved Concerns +**For FAO the two agree; for CRAF'd nobody can yet say.** `views-models/postprocessors/` contains `un_fao` and nothing else — there is **no `un_crafd` postprocessor directory**, so CRAF'd's historical `name=` has never been resolved, let alone compared against its consumer's filter. That makes this worse rather than better: for the live partner the agreement is a coincidence nobody wrote down, and for the new one it is an assumption that will first be tested by a production run. Whoever creates that directory decides, without knowing it, whether CRAF'd's actuals are retrievable. + +**Nothing in this repository asserts they agree.** `tests/test_product.py` asserts `CONSUMER_DOCUMENT_NAME` for the forecast leg; `tests/test_hop_b_sink_e2e.py` checks `consumer_name` on the forecast leg. Neither touches the historical leg's `name=`. A rename of the views-models directory — an ordinary, plausible act, done in a different repository by someone who has never read this file — silently detaches the historical artifact from the consumer's filter while every test here stays green and every delivery run reports success. + +This is ADR-003's rule broken in the quiet direction: the delivery **infers** its consumer identity from a path instead of reading the declaration that exists three lines away. It is also the fourth home for partner identity, where C-69's 2026-07-31 note counted three and recommended consolidation rather than relocation. Consolidation did not reach this line. + +**Scope note:** the crafd package inherited this unchanged from `unfao`; PR #211 did not introduce it, it doubled it. Registering it against both partners rather than against the PR. + +Cross-refs: **C-01** (RESOLVED — the metadata-completeness gate; same partner, same delivery, different field), **C-69** (RESOLVED — "partner identity has THREE homes"; this is the fourth and the note's consolidation recommendation is the fix), **C-33** (the duplication that turned one instance into two), ADR-013 §4.1a (F1 invisibility), ADR-003 (declarations over inference), #211. + +**RESOLVED 2026-08-04 (B1).** Both legs now name the document from the declaration: +`name=product.CONSUMER_DOCUMENT_NAME` replaces `name=self._model_path.model_name` in each +partner's historical upload, and `tests/test_product.py::test_both_delivery_legs_name_the_document_from_the_declaration` +asserts one forecast leg and one historical leg per partner, mutation-proven three ways +(revert one leg; add a third; stop declaring on the forecast leg). + +**Delivery-neutral, verified before changing anything.** `CONSUMER_DOCUMENT_NAME` is +`"un_fao"` and the views-models directory is `un_fao`, so `model_name` resolved to the same +string. No delivered byte changes for FAO; what changes is that the agreement is now a +declaration rather than a coincidence in another repository's filesystem. + +**Fixed now rather than when it broke, because it was about to be sprung.** views-models#333 +creates CRAF'd's launcher directory. Whoever named it would have decided, without knowing +it, whether CRAF'd's historical artifact was retrievable — and the failure mode is an empty +endpoint, not an error. The constraint was posted on that issue on 2026-08-04; this removes +the need for anyone to honour it. + +One residual, unchanged and not this entry's: the guard is a source scan, because the +managers cannot be instantiated without Appwrite env and a views-models path manager. That +is the standing pattern here and the reason **#18** exists. + +--- ### C-07: Undeclared direct runtime dependencies in pyproject.toml — RESOLVED @@ -795,6 +1446,19 @@ Cross-refs: C-57 (registry drift — the most likely way a coordinate goes wrong **RESOLVED 2026-08-03 by the pipeline-core 3.0.0 bump (C-44).** The auto-create-and-retry is gone: `create_bucket` appears **zero** times in `modules/appwrite/file.py`, and `:1406` now carries an explicit *"Fail loud, BEFORE any write, if a target container does not exist"* guard. A wrong or stale coordinate now fails instead of silently provisioning new production storage. Upstream views-pipeline-core C-228; verified in the installed wheel. +**Residual, and it is not ours to close: the fix is verified by inspection, not by probe.** This entry closed on *provisioning* — `create_bucket` is gone and I read the guard. The neighbouring **delete** path is a different question and views-pipeline-core **#333** ([þing-02 ledger row C5](https://github.com/views-platform/views-pipeline-core/issues/333), **OPEN**) is the probe that would answer it. Its three siblings — their #322, #331, #332 — all shipped in 3.0.0; #333 did not, because it is **blocked on the operator issuing a test key** (þing-02 G2 item f/h). + +What it probes is specific and is not covered by any check we own. The de-dup lookup is a **database** read; the verify step is a **storage** read; and entry to the delete branch requires *the lookup to have succeeded*. So a key with **database read and no bucket-file read** gets past the lookup, fails the verify, and reaches the delete — while a *wholly* read-restricted key fails benignly at the lookup and proves nothing. The dangerous asymmetry is what you get cutting a write-object key by **operation** rather than by **resource**. + +Two facts make that concrete for this repository rather than theoretical: + +1. **The path has already run 108 times in production on FAO's outbound bucket** — run-0's uploads — and the þing-02 verdict records it was benign *only because the files were readable* (`orð_dómr.md:294-298`). +2. **We cannot state our own key's scopes from evidence.** `docs/CLONING.md:131` records that this repository ran for months under a key named for pipeline-core and nobody could say what it was scoped to. So the dangerous shape cannot be ruled out by inspection here either. + +Nothing to do in this repo, and no reason to reopen the entry: the code fix is real and verified. Recorded because closing C-58 on the provisioning half should not read as closing the delete half, and because #333's gate — *"no scoped writer key is issued before C1 and C2 ship and C5's probe passes"* — is an operator action with this repository downstream of it. + +Cross-refs: **C-79** (`_ContractStorePort.upload` fails *open* on an unrecognised result — the same delete-adjacent surface), **C-40** (we run pipeline-core's client under our own identity, which is what þing-02 was about), views-pipeline-core #333/#322/#331/#332. + --- ### C-62: The pinned pipeline-core release still installs geopandas and torch into a repo that architecturally excised them — RESOLVED @@ -1104,6 +1768,10 @@ Verified 2026-08-02: `grep -rn "/home/" tests/ scripts/ views_postprocessing/ -- | Trigger | When treating `test_datafactory_deploy_readiness` as a release gate (it never runs in CI), or when a contributor's local `pytest` fails on it — re-promote / re-pin the strict-xfail now that views-datafactory has advanced to `1.5.0`-dev past its `v1.4.0` tag | | Location | `tests/test_datafactory_deploy_readiness.py` (`_DF = Path("/home/simon/.../views-datafactory")`, `skipif(not _DF.exists())`) | +**Overridden 2026-08-10 (ADR-016), and the objection was designed around rather than dismissed.** Sibling checkouts *were* added to the per-PR workflow. The recommendation's argument was specific — *"it couples this repo's CI to another repo's **default branch**, so an unrelated upstream commit turns this repo red"* — and every sibling checkout declares **`ref: main`** for exactly that reason. A commit on someone's feature branch, or on a default branch that is not `main` (views-appwrite's default is `development`), cannot reach us. `test_ci_sibling_coverage.py` makes `ref: main` a rule rather than a habit. + +**What is genuinely accepted, and should not be glossed:** a change merged to a sibling's `main` — a registry edition bump, say — *can* turn this repository red and block merges here until someone re-pins. That is not a defect being tolerated; it is the drift detector working, and the alternative is the state this entry was open about, where the drift was noticed only when a maintainer happened to run the suite. The cost is real and the trade is deliberate. + The cross-repo deploy-readiness gates introduced under C-36 are guarded by `skipif` on a **hardcoded local datafactory checkout path**, so they are **skipped in CI** and only ever execute on one developer's machine. There, `test_version_bumped_past_latest_tag` is currently **failing**: it is an `xfail(strict)` that flipped to XPASS because datafactory moved to `1.5.0`-dev past its `v1.4.0` tag — exactly the auto-flip C-36's resolution anticipated, but because of the hardcoded path the flip surfaces as a **local red** rather than a CI signal, and breaks local `pytest` runs (the suite is run with this test deselected). No correctness/reliability impact on the delivery → **Tier 4** (test hygiene). C-36 (resolved) converted these gates to strict-xfail but did not capture the local-path / CI-skip dimension. See also C-36 (the resolved strict-xfail conversion this extends), C-44 (the datafactory version-state coupling). @@ -1141,7 +1809,7 @@ The right axis was **exact equality on string constants**, not statement shape. **Gated, and honestly so.** The checks need a views-appwrite checkout and skip without one, naming `VIEWS_APPWRITE` and the conventional sibling path so a contributor can run them rather than merely watch them skip. The would-catch-a-rename proof runs in CI with no checkout at all. Resolution helper shared with **C-46** (S7) in `tests/conftest.py` — the second incident, which is this repo's named trigger for extracting. -**Residual — now tracked as C-81.** The gated half does not run in CI, which needs a views-appwrite checkout in the workflow. That was recorded here and in **C-46** as *"a CI-cost and cross-repo-coupling decision, not a code fix … worth deciding once for both"*, and it sat as a residual on two RESOLVED entries, which is where residuals go to be forgotten. It now has a live entry with a measured cost (17 tests, 9 of them new in this arc), a per-sibling answer, and an owner: **C-81**. views-appwrite is private, so it needs a token — an operator decision. +**Residual — RESOLVED 2026-08-10 (ADR-016).** The gated half did not run in CI, which needed a views-appwrite checkout in the workflow. It has one: that repository went public on 2026-08-08 and the workflow now fetches it, so these checks run on every pull request. That was recorded here and in **C-46** as *"a CI-cost and cross-repo-coupling decision, not a code fix … worth deciding once for both"*, and it sat as a residual on two RESOLVED entries, which is where residuals go to be forgotten. It now has a live entry with a measured cost (17 tests, 9 of them new in this arc), a per-sibling answer, and an owner: **C-81**. views-appwrite is private, so it needs a token — an operator decision. A second, smaller instance of the same shape: these checks parse TOML with `tomllib`, stdlib from Python 3.11, and `pyproject` declares `>=3.11`. CI runs 3.11 and executes them. The maintainer's box runs **3.10**, below the declared floor, so they skip there — the local suite is quietly weaker than a green `pytest -q` suggests. Not a repo defect and not worth its own entry; recorded because "a gate that does not run" is exactly what C-46 is open for, and the CI decision should cover both. | | Tier | 3 | @@ -1168,7 +1836,47 @@ This is ADR-014 §2 in its narrow form: a guard's *scope* is part of what has to PR #211 re-pins crafd to `1.4.1` / `90fc105` and parameterises **three** of the four checks over both partner declarations — names-and-class, pinned edition, commit reachability. The fourth, the value-copy scan, was never partner-scoped: it walks `_PKG.rglob("*.py")` and so covered `crafd/` from the day it landed. A third partner is now a one-line addition, and an unguarded one is a failure. This entry stays RESOLVED — the mechanism was right, its reach was not — but the residual below now has a companion: a detector that names its subject is a detector that will miss the next subject. -Cross-refs: C-74 (the guard this paragraph vouched for), C-33 (store identity still hardcoded per store — the same env surface, different concern), C-58 (what happens when a coordinate is wrong rather than missing), C-44 (the pipeline-core version coupling that would carry a registry change), issues #134/#135/#138 (this repo's discharged þing-01 obligations), #104 (README env block placeholders). +**⚠ AMENDED 2026-08-11 — the value-copy scan was blind in this repository's own house style, twice, and the second time is the general lesson.** + +The scan was extended to tracked markdown during the development→main sync, because README.md had carried four real coordinate values. Two reviews later, it was still missing two whole classes of copy — both of them ordinary markdown, both proven by injecting real registry values and watching the guard stay green: + +| form | why it was invisible | fixed | +|---|---|---| +| `NAME=value # comment` | `(.+?)\s*$` swallowed the comment into the captured value, so nothing compared equal. **README.md's own Configuration block is written in exactly this form.** | 2026-08-11 (iteration 2) | +| `` - `NAME=value` ``, `` \| NAME=value \| ``, `` set `NAME=value` before … `` | the pattern anchored the coordinate name at `^\s*`, so it saw only an assignment that *starts a line* | 2026-08-11 (iteration 3) | + +The shared cause is not a regex bug. **The pattern described one way of writing markdown — the way this repository happens to write it today — and a guard against publishing a value has to survive the next contributor writing a bullet instead of a fenced block.** The name may now be preceded by anything that is not part of an identifier, and the value ends at whatever terminates it in prose: a comment, a closing backtick, or a table pipe. Thirteen forms are proven caught; the four legitimate mentions that must stay silent are proven silent. + +**The residual, which is deliberate and should not be "fixed".** This remains a *syntax* match. A value merely named in a sentence — "the six stranded documents in ``" — is not a copy and does not fire. An earlier draft that matched any occurrence fired on a dozen documents, and the lesson recorded above applies to itself: a guard that cries wolf gets deleted, after which the real rule is unguarded (ADR-014 §3). The class this cannot catch is a value pasted into prose with no assignment syntax anywhere near it. That is accepted, because the alternative has been tried and was worse. + +**Mitigation — landed 2026-08-12 (#243). The parse is gone; the scan matches the pair.** + +The scan no longer parses a line and compares what it captured. It knows both halves before it starts — the declared names and the declared values — so it matches `NAME = VALUE` directly, with `findall` rather than one match per line. Every blindness recorded above came from the parse: the swallowed comment, the missing terminator, the six-character lookbehind, the second assignment on a table row. None is expressible now. + +The two halves need not correspond: `APPWRITE_X=` is the ordinary copy-paste slip and is reported as a copy, naming the coordinate that *declares* the value rather than the one the line assigns. + +**The `secret` exemption is gone.** It was an inline name literal under a comment saying the scope came from the declared partition and not from an inline list. Measured, `[secret]` carries no values today, so removing it changes nothing now and closes the hole the day one gains a value. + +**The stopping rule is a test, not a paragraph.** `test_the_scan_understands_every_assignment_form_this_repo_writes` enumerates every line in this repository's own tracked markdown that assigns a declared coordinate, and fails if the matcher cannot read one. So the form list is derived from the corpus: a form no document here uses is not a gap, and a new form is added in the same change as the document that introduces it. That is what stops the fifth widening. + +**The four scope mutations routed from #242 are all closed**, by one richer fixture rather than four tests: two copies, in two sections, one short, one in markdown. Each element defeats a specific narrowing — stopping at the first finding, reading only `target`, reinstating a length floor, skipping the markdown branch. Plus one small test for a package module that does not parse, which must be refused rather than stepped over. All five measured surviving before, all five caught after. + +**And an exclusion list was built and then deleted**, which is the useful part. The plan called for excluding values spelled like this repository's own code. Measured against what the AST branch actually sees — non-docstring string constants — **the collision does not exist**: the only registry values appearing that way are the two `contract` rows, and `contract` is MIRRORED rather than CONSUMED, so it is never in the scanned set. The list narrowed a security scan for a problem this branch does not have. The residual is latent and recorded in C-97: the day someone writes a package directory name as a bare string constant, the scan will fire on it. + +**Why this belongs on C-57 rather than in a new entry.** It is the same guard, the same failure direction, and the same lesson this entry already records one layer down: the earlier amendment found the scan's *scope* was never mutation-proven (it named `unfao` and missed `crafd`); this one finds its *matching* was never proven against the file formats it scans. Scope, matching, and now syntax-variant — three ways for a mutation-proven guard to be proven against the wrong thing. + +**⚠ CORRECTED 2026-08-12, and the correction is the same mistake one level up.** The amendment immediately above claims "thirteen forms are proven caught" and cites `` set `NAME=value` before … `` as evidence the mid-sentence class is covered. An independent review supplied twenty-nine forms and **fifteen missed**, including the *unbackticked* form of that very example: `set NAME=value before running` captures `'value before running'`. The cited case passes only because of its backticks. Measured, not argued. + +Two distinct defects sit under those fifteen, and both are the pattern-describes-one-dialect cause this entry already names: + +1. **The unquoted value has no whitespace terminator.** It ends only at `#`, a backtick or `|`, so any value followed by prose escapes. And `search` is called once per line, so on a two-variable table row every assignment after the first is invisible. +2. **The boundary is an allow-list, not a predicate.** The comment above it says the name may be preceded by "anything that is not part of an identifier"; the code is a six-character lookbehind. A quoted, parenthesised or colon-prefixed assignment produces no match at all — so the pattern can *parse* a quoted value but cannot *see* a quoted assignment. + +**And a fourth failure direction, opposite to the other three.** Removing the six-character floor admitted the registry's two shortest values into the ban-set. Measured: **they are this repository's own package directory names — the exact contents of `PARTNER_PACKAGES`** — matched by exact string equality against every non-docstring string constant under `views_postprocessing/`. The justification given, "measured, removing it keeps the suite green", measured today's source and not the next commit's. The next `logging.getLogger()` or `Literal[, …]` fails a security guard for code that copied nothing, and this entry's own doctrine says what happens next. It also hands views-appwrite a way to redden this repository by declaring any short ordinary word as a value — which is **C-86**'s coupling, arriving through the guard that was supposed to reduce it. + +Registered as **C-89** (the leak half) and tracked with this entry's remediation. **C-93** records why the author's own proof did not find any of it. + +Cross-refs: C-74 (the guard this paragraph vouched for), C-33 (store identity still hardcoded per store — the same env surface, different concern), C-58 (what happens when a coordinate is wrong rather than missing), C-44 (the pipeline-core version coupling that would carry a registry change), **C-86** (the drift checks this scan sits beside), issues #134/#135/#138 (this repo's discharged þing-01 obligations), #104 (README env block placeholders). --- @@ -1218,7 +1926,7 @@ Cross-refs: C-15 (the provenance this field serves), C-22 (the recall process th Replaced by tests of `contract/historical.assert_metadata_complete` — the code that actually gates a delivery — parametrised over the **imported** `METADATA_COLS`, plus source-scan pins that the gate stays at build time and is still invoked. Verified 2026-08-02: `pytest -q tests/test_validation.py` → **14 passed**. Mutation-tested: narrowing the gate to a single column fails **9 of 14**; the old suite passed that mutation untouched, because it was not testing the gate. -**Residual 2 — the enrich→validate end-to-end test — RELOCATED to #18**, per the Register Conventions' relocation rule (a relocation is not complete until the destination exists and is cited by number). Every *leg* is now covered — enrichment (`test_enrichment.py`, 16), artifact build (`test_historical_builder.py`, 7), reader parity (`test_historical_parity.py`, 3), the invariants on primitives (`test_input_integrity_e2e.py`, 8), the wire end to end (`test_hop_b_sink_e2e.py`, 6), the null-gate (`test_validation.py`, 14). What remains uncovered is **the manager orchestrating them**, which needs views-pipeline-core and a production-like Appwrite environment — and þing-02 **D2** forbids integration tests against the production project, no non-production one existing. +**Residual 2 — the enrich→validate end-to-end test — RELOCATED to #18**, per the Register Conventions' relocation rule (a relocation is not complete until the destination exists and is cited by number). Every *leg* is now covered — enrichment (**no longer a leg**: `GaulLookupEnricher` and `test_enrichment.py` were deleted in #90/B3b, the lookup join having moved into the frame build; recorded here because the closure above was argued from a list this deletion shortened), artifact build (`test_historical_builder.py`, 7), reader parity (`test_historical_parity.py`, 3), the invariants on primitives (`test_input_integrity_e2e.py`, 8), the wire end to end (`test_hop_b_sink_e2e.py`, 6), the null-gate (`test_validation.py`, 14). What remains uncovered is **the manager orchestrating them**, which needs views-pipeline-core and a production-like Appwrite environment — and þing-02 **D2** forbids integration tests against the production project, no non-production one existing. That gap has **two standing trackers already**, which is why keeping a third here is noise rather than signal: issue **#18** (open since 2026-06-04) and `tests/test_falsification_campaign_3_5.py`, an `xfail(strict)` probe that **flips to XPASS the moment someone writes the test** — a self-surfacing tracker, which is more than this entry was doing. | | Tier | 3 | diff --git a/scripts/build_gaul_lookup.py b/scripts/build_gaul_lookup.py index 46eb21d..955e89e 100644 --- a/scripts/build_gaul_lookup.py +++ b/scripts/build_gaul_lookup.py @@ -28,8 +28,9 @@ import os from pathlib import Path -import pandas as pd +import numpy as np import pyarrow as pa +import pyarrow.compute as pc import pyarrow.parquet as pq from views_postprocessing.contract.gaul_schema import ( @@ -76,21 +77,21 @@ def _resolve_datafactory() -> Path: return sibling -def _load_source(datafactory: Path) -> pd.DataFrame: - """Join the 7 GAUL parquets on gid into one wide frame (source names).""" +def _load_source(datafactory: Path) -> pa.Table: + """Join the 7 GAUL parquets on gid into one wide table (source names). + + Arrow throughout since #90 — the sources are parquet, the output is parquet, and + pandas was only ever the thing in the middle. The join is an inner join on ``gid``: + a cell missing from any one source has no complete row to contribute, and the + completeness filter in ``build`` would drop it anyway. + """ gaul_dir = datafactory / "data" / "raw" / "gaul_admin" - frames = {} + table = None for src_col in SOURCE_RENAME: - t = pq.read_table(gaul_dir / f"{src_col}.parquet") - s = pd.Series( - t.column("value").to_pylist(), - index=t.column("gid").to_pylist(), - name=src_col, - ) - frames[src_col] = s - df = pd.DataFrame(frames) - df.index.name = "gid" - return df + one = pq.read_table(gaul_dir / f"{src_col}.parquet").select(["gid", "value"]) + one = one.rename_columns(["gid", src_col]) + table = one if table is None else table.join(one, keys="gid", join_type="inner") + return table def _region_gids(datafactory: Path, region: str) -> set[int] | None: @@ -197,46 +198,82 @@ def _provenance(datafactory: Path, *, datasets: tuple[str, ...]) -> dict: return out -def build(datafactory: Path, region: str, out: Path) -> pd.DataFrame: +def build(datafactory: Path, region: str, out: Path) -> pa.Table: + """Build the lookup and write it. Arrow end to end (#90). + + **What must not change, and is asserted by the fidelity suite:** the nine declared + columns in ``METADATA_COLS`` order with ``priogrid_gid`` last, the four name columns + dictionary-encoded, codes ``int64``, coordinates ``float64``, rows sorted by cell id, + and the declared metadata keys. The delivered artifact is read as arrow by its only + consumer (``contract.gaul_lookup.load``), so those are the observable contract. + + **One deliberate difference from the pandas build:** the written file no longer + carries a ``pandas`` schema-metadata blob. It described an index that arrow does not + have, nothing in this repository reads the lookup with pandas, and reproducing it + would have meant keeping knowledge of pandas' metadata format in the one script this + change exists to remove pandas from. + """ src = _load_source(datafactory) # Optionally restrict to a region's cell set. region_gids = _region_gids(datafactory, region) if region_gids is not None: - src = src.loc[src.index.intersection(sorted(region_gids))] - - # Rename to the contract names. - df = src.rename(columns=SOURCE_RENAME) - - # Keep only fully-complete cells. Incomplete cells must NOT enter the - # lookup: an unknown/incomplete gid then merges to null downstream and the - # manager's _validate() gate crashes (fail-loud) instead of shipping a hole - # or a -1 sentinel. Never carry -1 / "" as a value. - complete = pd.Series(True, index=df.index) - for c in CODE_COLS: - complete &= df[c].notna() & (df[c] != -1) - for c in NAME_COLS: - complete &= df[c].notna() & (df[c].astype(str).str.len() > 0) - dropped = int((~complete).sum()) - df = df[complete].copy() - - # Coordinates from the gid (no geometry needed). - gids = df.index.to_numpy() - df["pg_xcoord"] = [xcoord(int(g)) for g in gids] - df["pg_ycoord"] = [ycoord(int(g)) for g in gids] + wanted = pa.array(sorted(region_gids), type=src.column("gid").type) + src = src.filter(pc.is_in(src.column("gid"), value_set=wanted)) + + # Rename to the contract names, keeping `gid` alongside. + table = src.rename_columns( + ["gid"] + [SOURCE_RENAME[c] for c in src.column_names if c != "gid"] + ) if src.column_names[0] == "gid" else src.rename_columns( + [SOURCE_RENAME.get(c, c) for c in src.column_names] + ) - # dtypes: codes numeric, coords float64, names/iso categorical (C-32 memory). + # Keep only fully-complete cells. Incomplete cells must NOT enter the lookup: an + # unknown/incomplete gid then gathers to null downstream and the delivery's null + # gate fails loud instead of shipping a hole or a -1 sentinel. Never carry -1 / "". + # Typed explicitly: on an empty table `pa.array([True] * 0)` infers NULL type, and + # `pc.and_` then raises ArrowNotImplementedError before the zero-row guard below can + # say anything useful. Found by the guard's own mutation test. + complete = pa.array([True] * table.num_rows, type=pa.bool_()) for c in CODE_COLS: - df[c] = df[c].astype("int64") - for c in COORD_COLS: - df[c] = df[c].astype("float64") + col = table.column(c) + complete = pc.and_(complete, pc.and_(pc.is_valid(col), pc.not_equal(col, -1))) for c in NAME_COLS: - df[c] = df[c].astype("category") + col = table.column(c).cast(pa.string()) + complete = pc.and_( + complete, + pc.and_(pc.is_valid(col), pc.greater(pc.utf8_length(col), 0)), + ) + dropped = int(pc.sum(pc.invert(complete)).as_py() or 0) + table = table.filter(complete) + + # Coordinates from the gid (no geometry needed). Vectorised over the id array + # rather than a Python loop per row — the formula is the declared one either way. + gids = table.column("gid").to_numpy(zero_copy_only=False).astype(np.int64) + table = table.append_column( + "pg_xcoord", pa.array([xcoord(int(g)) for g in gids], type=pa.float64()) + ).append_column( + "pg_ycoord", pa.array([ycoord(int(g)) for g in gids], type=pa.float64()) + ) - df = df[METADATA_COLS] - df.index = df.index.astype("int64") - df.index.name = "priogrid_gid" - df = df.sort_index() + # dtypes: codes int64, coords float64, names dictionary-encoded. The dictionary + # encoding is the artifact's shipped type — it was `category` under pandas and the + # committed file carries dictionary. + cast = {c: pa.int64() for c in CODE_COLS} + cast.update({c: pa.float64() for c in COORD_COLS}) + columns, names = [], [] + for c in METADATA_COLS: + col = table.column(c) + if c in NAME_COLS: + col = pc.dictionary_encode(col.cast(pa.string())) + else: + col = col.cast(cast[c]) + columns.append(col) + names.append(c) + columns.append(table.column("gid").cast(pa.int64())) + names.append("priogrid_gid") + table = pa.Table.from_arrays(columns, names=names) + table = table.sort_by([("priogrid_gid", "ascending")]) # Hard invariants — the lookup must be clean by construction. # @@ -245,14 +282,20 @@ def build(datafactory: Path, region: str, out: Path) -> pd.DataFrame: # identical on disk (register C-61). The -1 check in particular has no downstream # backstop — -1 is non-null, so every gate in the delivery chain would pass it # straight through to FAO, which is exactly the resolved C-35 defect recurring. - if not df.index.is_unique: - dupes = df.index[df.index.duplicated()].unique().tolist() + key = table.column("priogrid_gid") + if len(pc.unique(key)) != table.num_rows: + counts = key.value_counts() + dupes = [ + counts.field("values")[i].as_py() + for i in range(len(counts)) + if counts.field("counts")[i].as_py() > 1 + ] raise LookupBuildError( - f"{len(dupes)} duplicate gid(s) in the lookup index: {dupes[:10]}. " - "A duplicated key multiplies rows through the enricher's left-merge with " - "every value non-null, so no downstream gate can see it (C-59)." + f"{len(dupes)} duplicate gid(s) in the lookup key: {dupes[:10]}. " + "A duplicated key multiplies rows through a keyed gather with every value " + "non-null, so no downstream gate can see it (C-59)." ) - n_null = int(df.isna().sum().sum()) + n_null = sum(table.column(c).null_count for c in table.column_names) if n_null: raise LookupBuildError( f"lookup contains {n_null} null value(s); only fully-complete cells may " @@ -260,20 +303,31 @@ def build(datafactory: Path, region: str, out: Path) -> pd.DataFrame: "downstream." ) for c in CODE_COLS: - n_sentinel = int((df[c] == -1).sum()) + n_sentinel = int(pc.sum(pc.equal(table.column(c), -1)).as_py() or 0) if n_sentinel: raise LookupBuildError( f"{c} contains {n_sentinel} -1 sentinel(s). -1 is non-null, so it would " "reach FAO through every gate as a country/admin code (cf. C-35)." ) + # Register C-76: a build that filtered every cell out is not a result. Downstream + # this fails late and confusingly — `build_historical_table` raises about missing + # geography rather than about an empty lookup, and by then the artifact is + # committed. Two lines here, at the moment a human is present. + if table.num_rows == 0: + raise LookupBuildError( + f"the build produced ZERO cells for region {region!r}. Either the region " + "filtered every cell out, or the join found no overlap between the seven " + "source parquets. An empty lookup is writable and looks like a result; it " + "is not one." + ) out.parent.mkdir(parents=True, exist_ok=True) - table = pa.Table.from_pandas(df, preserve_index=True) - meta = dict(table.schema.metadata or {}) - meta[b"adr"] = b"ADR-011" - meta[b"region"] = region.encode() - meta[b"n_cells"] = str(len(df)).encode() - meta[b"n_dropped_incomplete"] = str(dropped).encode() + meta = { + b"adr": b"ADR-011", + b"region": region.encode(), + b"n_cells": str(table.num_rows).encode(), + b"n_dropped_incomplete": str(dropped).encode(), + } prov = _provenance(datafactory, datasets=(AREA_MAJORITY_DATASET, stamp_dataset(region))) # The DECLARED stamp: one flat key, composed here, read verbatim by the consumer # (C-60). Key order in parquet metadata carries no meaning; this is a dict. @@ -282,9 +336,9 @@ def build(datafactory: Path, region: str, out: Path) -> pd.DataFrame: table = table.replace_schema_metadata(meta) pq.write_table(table, out) - print(f"region={region} cells={len(df):,} dropped_incomplete={dropped:,}") + print(f"region={region} cells={table.num_rows:,} dropped_incomplete={dropped:,}") print(f"wrote {out} ({out.stat().st_size/1e6:.2f} MB)") - return df + return table def main() -> None: diff --git a/tests/conftest.py b/tests/conftest.py index 78493ac..1943321 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,23 @@ -"""Shared test fixtures — currently just one: how to find a sibling repository. +"""Facts several test modules share, and the helpers that read them. -**Why this exists (S6 / #187, S7 / #188; register C-46, C-57).** Three places needed a -views-platform sibling checkout and each found it differently: +**Read this first if you are new.** Despite the name, almost nothing here is a pytest +fixture. The file holds four separate things: + +1. **which packages exist** — ``PARTNER_PACKAGES`` and ``MACHINERY_PACKAGES``, the + two kinds of top-level package under ``views_postprocessing/``; +2. **the sibling repositories** — ``Sibling``/``SIBLINGS`` plus three functions for + locating one on disk. A *sibling* is another repository in the views-platform + organisation that sits beside this one in a developer's folder — never something we + install; +3. **which repository consumes which partner's delivery** — ``CONSUMER_REPO``; +4. **two read-only git helpers** used by cross-repository checks. + +Those four are not obviously one file's worth of responsibility, and that is recorded +rather than defended: register **C-88**, with the trigger for splitting it. Read it +before adding a fifth thing here. + +**Why the sibling helpers exist.** Three places needed to locate a views-platform +sibling checkout and each did it differently: - ``scripts/build_gaul_lookup.py`` resolved ``$VIEWS_DATAFACTORY``, then the sibling directory, with a ``--datafactory`` override and a fail-loud message naming both — @@ -26,6 +42,7 @@ import os import subprocess +from dataclasses import dataclass from pathlib import Path import pytest @@ -60,15 +77,99 @@ #: either a partner or machinery, and the same test refuses to let it be neither. MACHINERY_PACKAGES = ("contract", "delivery") -#: repo name -> the environment variable that overrides its location. -#: Declared, never derived from the name: ``views-datafactory`` → ``VIEWS_DATAFACTORY`` -#: happens to be mechanical, but a future sibling need not follow the pattern and -#: guessing it would be the inference ADR-003 forbids. -SIBLING_ENV = { - "views-datafactory": "VIEWS_DATAFACTORY", - "views-appwrite": "VIEWS_APPWRITE", - "views-faoapi": "VIEWS_FAOAPI", - "views-crafdapi": "VIEWS_CRAFDAPI", +@dataclass(frozen=True, kw_only=True) +class Sibling: + """What this repository declares about one views-platform sibling (ADR-016). + + Three fields. ``env`` says where to find this repository on disk; ``ci_checkout`` + says whether CI fetches it; ``note`` says why not, whenever the answer is no. + + **An earlier version also carried ``public`` and ``public_checked``.** They were + removed on 2026-08-10 after a review found them circular: ``public`` was read by + exactly one rule, and that rule existed to protect ``public``'s verifiability. + Nothing else consulted either field, and deleting both changed no behaviour. Whether + a sibling is public is now simply part of ``note`` — prose, where it belongs, because + nothing here could verify it anyway. + + **Keyword-only and frozen.** A plain dict would let a missing ``ci_checkout`` read as + ``None`` and silently exempt that sibling from every rule; that is the failure this + repository has registered more often than any other. Here the omission is a + ``TypeError`` at import. + + **No validation in ``__post_init__``.** See ``broken_sibling_overrides`` below for + what raising at import time costs: one typo became three collection errors and zero + tests run. The rules live in ``tests/test_ci_sibling_coverage.py``. + """ + + #: The environment variable that overrides this sibling's location. Declared, never + #: derived: ``views-datafactory`` → ``VIEWS_DATAFACTORY`` happens to be mechanical, + #: but a future sibling need not follow the pattern and guessing it would be the + #: inference ADR-003 forbids. + env: str + #: Whether CI fetches this sibling. + ci_checkout: bool + #: Why not, when it is not fetched. Required in that case, and must name a record so + #: the non-coverage has an owner rather than a shrug. + note: str = "" + + +#: The views-platform repositories whose current state this repository's tests read. +#: +#: `views-appwrite` was private until **2026-08-08**, when it was deliberately made +#: public (`views-appwrite@9d80b75`, "docs: record going public"). The workflow comment +#: here went on saying PRIVATE afterwards, which is the whole argument for declaring the +#: fact with a date instead of narrating it in prose. +SIBLINGS = { + "views-datafactory": Sibling( + env="VIEWS_DATAFACTORY", + ci_checkout=False, + note=( + "PUBLIC, but its checks need the producer's raw GAUL parquets " + "(data/raw/gaul_admin/*.parquet), which are NOT in its git repository. " + "Checking it out converts an honest skip into a FileNotFoundError — measured " + "2026-08-03, tried and reverted. Closing this needs the data published " + "somewhere fetchable, not an access grant. Register C-46." + ), + ), + "views-appwrite": Sibling( + env="VIEWS_APPWRITE", + ci_checkout=True, + note=( + "PUBLIC. Fetched, and the fetch is load-bearing rather than convenient: it " + "carries the coordinate-registry drift checks, and under ADR-017 it becomes " + "the authority source for the delivery-label check too. Pruning it as an " + "unused sibling would disable both and leave a green build — the invisible " + "skip ADR-016 §6 exists to prevent." + ), + ), + "views-faoapi": Sibling( + env="VIEWS_FAOAPI", + ci_checkout=False, + note=( + "PRIVATE — the only one, and that is why it is not fetched. Checking it " + "out needs a credential, which is an " + "operator decision deferred pending a request to FAO to make the repository " + "public. **Nothing is dark because of this any more** (2026-08-11): ADR-017 " + "moved the delivery-label check onto the public coordinate registry, which " + "needs no credential, so neither side reads the other. This entry stays " + "PRIVATE because the fact is still true, not because anything is blocked on " + "it. See ADR-017 and register C-81." + ), + ), + "views-crafdapi": Sibling( + env="VIEWS_CRAFDAPI", + ci_checkout=False, + note=( + "PUBLIC, and no longer fetched. It served exactly one test — the check that " + "read this consumer's source for its query filter — and #248 deleted that " + "check rather than repairing it: the same read broke twice in 24 hours " + "because both consumers refactored a literal into a named constant, which " + "ADR-017 §7 predicted. The declaration stays because CONSUMER_REPO still " + "names this repository; only the fetch is gone. What the source read used to " + "cover is register C-92, and views-crafdapi#55 is the ask that would close it " + "where the fact lives." + ), + ), } #: partner package -> the repository that CONSUMES its delivery. @@ -78,10 +179,12 @@ #: the second one ``un-crafdapi`` while the repository on disk is ``views-crafdapi`` — #: exactly the kind of near-miss that makes guessing expensive. #: -#: This exists so the consumer-document-name pin can be checked **across the seam** -#: rather than asserted locally. A name this repo declares and the consumer filters on -#: is a fact this repo does not own; declaring it here is right, but only the sibling -#: checkout can confirm it still matches (ADR-014 §1 — the guarantee needs a check). +#: It exists so this repository records **who receives each delivery**. It used to also +#: locate a sibling checkout so the consumer-document-name pin could be read from that +#: consumer's source; #248 deleted that read (ADR-017 §7 — we were never entitled to +#: depend on another repository's file layout, and two consumers proved it in a day by +#: improving theirs). What the map is for now is addressing: it is who register C-92's +#: cross-repo asks are sent to. CONSUMER_REPO = { "unfao": "views-faoapi", "crafd": "views-crafdapi", @@ -97,12 +200,12 @@ def sibling_repo(name: str) -> Path | None: Returns ``None`` rather than raising so callers can skip; a missing sibling is a normal condition in CI, where only this repo is checked out. """ - if name not in SIBLING_ENV: + if name not in SIBLINGS: raise KeyError( f"no environment variable declared for sibling {name!r}; add it to " - f"SIBLING_ENV rather than guessing one from the name" + f"SIBLINGS rather than guessing one from the name" ) - override = os.environ.get(SIBLING_ENV[name]) + override = os.environ.get(SIBLINGS[name].env) candidate = Path(override) if override else _REPO.parent / name return candidate if candidate.exists() else None @@ -122,12 +225,24 @@ def broken_sibling_overrides() -> dict[str, str]: ``Interrupted: 3 errors during collection`` and **zero tests run** — trading silent under-coverage for total loss of the suite. One clean failure says the same thing and lets the other 360 tests report. + + **An EMPTY variable is the case this missed, and it is the one CI produces.** The + filter was ``if value and ...``, so ``VIEWS_APPWRITE=""`` read as unset: + ``sibling_repo`` fell through to the conventional ``../views-appwrite``, which does + not exist in a CI workspace, and seven checks skipped on a green build. A YAML + interpolation that resolves to nothing — ``${{ env.TYPO }}`` — produces exactly that + empty string, so this is the *likely* misconfiguration in CI, not an exotic one. + + ``.strip()`` and not merely ``if value is not None`` because **``Path("").exists()`` + is ``True``** — it resolves to the current directory. Dropping the truthiness test + without the strip would report an empty override as a perfectly good checkout, which + is worse than the bug being fixed. """ return { var: value - for var in SIBLING_ENV.values() + for var in (s.env for s in SIBLINGS.values()) for value in [os.environ.get(var)] - if value and not Path(value).exists() + if value is not None and (not value.strip() or not Path(value).exists()) } @@ -141,7 +256,7 @@ def require_sibling(name: str) -> Path: path = sibling_repo(name) if path is None: pytest.skip( - f"{name} checkout not found — set {SIBLING_ENV[name]}=/path/to/{name}, " + f"{name} checkout not found — set {SIBLINGS[name].env}=/path/to/{name}, " f"or place it alongside this repo at {(_REPO.parent / name)}" ) return path diff --git a/tests/seam_registry.py b/tests/seam_registry.py new file mode 100644 index 0000000..d6a4f90 --- /dev/null +++ b/tests/seam_registry.py @@ -0,0 +1,153 @@ +"""Reading the platform coordinate registry, and refusing to read it badly. + +The registry is a TOML file in **views-appwrite** declaring every cross-repository +coordinate on the delivery seam. This repository never copies it — it is referenced, and +read from a sibling checkout when one is present. + +**Why this is its own module.** Two test modules now need it: `test_env_declaration.py` +for the coordinate sections, `test_product.py` for the `[contract.*]` delivery labels. +It began as one reader, was hand-copied into the second, and the copies immediately +disagreed — one read the sibling's `main`, the other whatever branch the clone sat on, +which is the very defect a review had just caught in the first. That is the second +incident, which is this repository's stated trigger for extracting (WET before DRY: two +copies that are understood beat one abstraction that is guessed — until the second copy +proves the shape). + +It is deliberately **not** in `conftest.py`. That file already holds four unrelated +groups and register **C-88**'s trigger is a fifth arriving. + +**What this module refuses, and why each refusal exists.** Every one is a real accident +that a silent read would have turned into a permanently green test: + +- a ref that does not resolve, or resolves to something that is not a commit; +- a ref whose resolved sha does not begin with it — an annotated **tag**, which git peels + silently and which yields a 404 from the blob URL these modules publish; +- an **empty** file: `tomllib.loads("")` is `{}`, and an empty registry compared against + an empty projection passes forever; +- a file that parses but carries no `meta.version` and no `connection` rows, i.e. is not + the registry; +- a parse error, wrapped rather than raised raw — a helper whose justification is failing + legibly must not hand back a bare traceback. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +#: The registry, relative to a views-appwrite checkout. +REGISTRY_RELPATH = Path("docs") / "ADRs" / "platform" / "coordinate_registry.toml" + +#: A row that carries no value at all — distinct from any string a registry could hold. +ABSENT = object() + + +class RegistryReadError(RuntimeError): + """The registry could not be read — never a silently empty dict.""" + + +def registry_at(repo: Path, ref: str) -> dict: + """The registry as it stood at ``ref``. Raises rather than returning ``{}``. + + ``ref`` must name a **frozen commit**, and that has no exceptions. An empty ref reads + the git index; a branch name reads a moving tip. Either makes a pin-versus-current + comparison compare the registry to itself and report green against every future + edition — which is the trap this module exists for, so an exemption "just for the + caller that needs a branch" reopens it. Callers wanting `main` resolve it to a sha + first; :func:`registry_current` does exactly that. + """ + tomllib = pytest.importorskip("tomllib", reason="stdlib from 3.11; pyproject requires it") + + resolved = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"], + capture_output=True, text=True, check=False, timeout=30, + ) + if resolved.returncode != 0 or not resolved.stdout.strip(): + raise RegistryReadError( + f"{ref!r} does not resolve to a commit in {repo}. A pin must name a frozen " + "commit: an empty ref reads the index and a branch reads a moving tip, and " + "either would make the comparison compare the registry to itself." + ) + if not resolved.stdout.strip().startswith(ref): + raise RegistryReadError( + f"{ref!r} resolves to commit {resolved.stdout.strip()[:12]}, which does not " + "start with it — that is what an annotated TAG looks like. git peels tags " + "silently, so every check here would pass while the blob URL these modules " + "publish returns 404. Pin the commit." + ) + + result = subprocess.run( + ["git", "-C", str(repo), "show", f"{ref}:{REGISTRY_RELPATH.as_posix()}"], + capture_output=True, text=True, check=False, timeout=30, + ) + if result.returncode != 0: + raise RegistryReadError( + f"cannot read the registry at {ref!r} in {repo}: git exited " + f"{result.returncode} ({result.stderr.strip()[:200]}). A pinned commit that " + "cannot be read is not a baseline — run `git fetch` in that checkout." + ) + if not result.stdout.strip(): + raise RegistryReadError( + f"the registry is EMPTY at {ref!r} in {repo}. Treating that as a registry " + "with no coordinates would make every comparison against it pass." + ) + try: + registry = tomllib.loads(result.stdout) + except Exception as exc: + raise RegistryReadError( + f"the registry at {ref!r} in {repo} did not parse as TOML: {exc}. This is " + "what happens when the path names a directory, or the blob is not the file " + "we think it is." + ) from exc + if not registry.get("meta", {}).get("version") or not registry.get("connection"): + raise RegistryReadError( + f"the registry at {ref!r} parsed but carries no meta.version or no connection " + "rows. Every real edition has both, so this is not the file we think it is — " + "refusing rather than comparing against it." + ) + return registry + + +def registry_current(repo: Path) -> dict: + """The registry as the sibling's ``main`` has it — never its working tree. + + A sibling clone normally sits on whatever branch its own agent last worked on. + Comparing against that grades this repository on unreviewed content, or reddens it on + a branch nobody merged — issue #196 verbatim, the case that cost this platform a + withdrawn pull request. The `ref: main` discipline lived only inside the CI workflow; + a developer running the suite locally had no such thing. + + Read errors propagate unchanged, so a registry that is corrupt or anchorless **on + main** says so, rather than being reported as "neither ref resolves". + """ + head = "" + for ref in ("origin/main", "main"): + resolved = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"], + capture_output=True, text=True, check=False, timeout=30, + ) + if resolved.returncode == 0 and resolved.stdout.strip(): + head = resolved.stdout.strip() + break + if not head: + raise RegistryReadError( + f"neither origin/main nor main resolves in {repo}. The comparison is against " + "that repository's ratified content, so there is nothing to compare against " + "— run `git fetch` there." + ) + return registry_at(repo, head) + + +def rows(registry: dict, sections: tuple) -> dict[str, tuple]: + """``name -> (section, class, value-or-ABSENT)`` across ``sections``. + + The one shared projection. Two hand-copied versions had already diverged on null + handling — one raised ``AttributeError`` on a null section, the other did not. + """ + return { + name: (section, body.get("class"), body.get("value", ABSENT)) + for section in sections + for name, body in (registry.get(section) or {}).items() + } diff --git a/tests/test_ci_sibling_coverage.py b/tests/test_ci_sibling_coverage.py new file mode 100644 index 0000000..ee0238a --- /dev/null +++ b/tests/test_ci_sibling_coverage.py @@ -0,0 +1,506 @@ +"""The workflow and the sibling declaration must agree — ADR-016. + +**The failure this exists to prevent has already happened.** A comment in +`run_pytest.yml` stated that `views-appwrite` was private. It went public on 2026-08-08 +and the comment did not, so seven cross-repository checks stayed dark in CI for no reason +at all — and the document written to justify a credential for them was drafted the next +day, against a fact that had already changed. + +The fix is not a better comment. It is that **which siblings CI checks out is declared +once**, in `tests/conftest.py::SIBLINGS`, and this file fails when the workflow and that +declaration disagree — in either direction. + +**Whether a sibling is public is not modelled here, and that is deliberate.** An earlier +version carried a `public` flag and a rule pairing it with the presence of a checkout +token. A review found the pair circular — the field was read by exactly one rule, and +that rule existed to protect the field's verifiability — so both were removed on +2026-08-10 without any change in behaviour. Visibility now lives in a sibling's `note`, +as prose, because nothing here could verify it in any case: no test in this suite touches +the network. + +What still holds without the flag: a checkout of a repository CI cannot read simply fails +the build, because the default `GITHUB_TOKEN` is scoped to this repository. Rule G5 keeps +that failure loud. + +**Why the rules run G1, G2, G4, G5, G6, G7 with two numbers missing.** G3 paired a +sibling's visibility against whether its checkout used a credential; G8 checked that a +recorded date was plausible. Both were deleted on 2026-08-10 after a review found them +circular — each existed to protect a field that existed to feed it. The survivors keep +their original names rather than being renumbered, so anything written about "G6" +elsewhere still means this G6. The gap is deliberate; nothing is missing. + +**Two sections of this file have inverted failure semantics, and that trips people.** +Everything above `── the rules, against the real workflow ──` fails when *the workflow or +the declaration* is wrong. Everything below `── the rules, against synthetic mutants ──` +fails when **the rule itself** is wrong — those tests feed a deliberately broken world to +a rule and demand it complain. If one of those fails, do not go looking at +`run_pytest.yml`; the workflow is fine and the guard has stopped guarding. + +**Every rule is a pure function of (workflow, siblings)**, so each can be run against a +synthetic mutant rather than only against the real file. A guard that can only be +demonstrated by editing CI is a guard nobody ever watches fail (ADR-014 §2) — this repo +already answers that three times over (`test_the_drift_check_would_catch_a_rename`, +`test_the_guard_would_actually_catch_a_violation`, `test_the_ban_actually_fires_on_a_living_doc`). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from tests.conftest import SIBLINGS, Sibling + +_WORKFLOWS = Path(__file__).resolve().parent.parent / ".github" / "workflows" +_TEST_WORKFLOW = _WORKFLOWS / "run_pytest.yml" + +#: Where a sibling checkout must land. Not tidiness: `pyproject.toml` excludes +#: `_siblings` from ruff, and the Lint step reported 745 findings in views-crafdapi's own +#: test suite the first time a sibling was checked out anywhere else. +_SIBLING_PATH_PREFIX = "_siblings/" + +#: The org every sibling belongs to. A `repository:` outside it is not a sibling. +_ORG = "views-platform/" + + +# ── parsing ────────────────────────────────────────────────────────────────── + + +def _load(path: Path) -> dict: + return yaml.safe_load(path.read_text()) or {} + + +def _steps(workflow: dict) -> list[dict]: + """Every step of every job, flattened.""" + return [ + step + for job in (workflow.get("jobs") or {}).values() + for step in (job.get("steps") or []) + if isinstance(step, dict) + ] + + +def _sibling_steps(workflow: dict) -> list[dict]: + """Checkout steps that fetch ANOTHER repository. + + Keyed on the presence of ``repository:``, not on ``uses: actions/checkout``. This + repo's own checkout is an `actions/checkout` step with no ``repository:``, so a rule + scoped to the action would fire on line one and be deleted the same day (ADR-014 §3). + """ + return [ + step + for step in _steps(workflow) + if str(step.get("uses", "")).startswith("actions/checkout") + and isinstance(step.get("with"), dict) + and step["with"].get("repository") + ] + + +def _repo_name(step: dict) -> str: + """``views-platform/views-appwrite`` -> ``views-appwrite``.""" + return str(step["with"]["repository"]).removeprefix(_ORG) + + +def _pytest_step(workflow: dict) -> dict | None: + """The step that actually runs the suite — where the env vars must land. + + Found by what it runs, not by its name. An `env:` block on the Lint step satisfies a + naive "the variable is set somewhere" check while the tests see nothing. + """ + for step in _steps(workflow): + if "pytest" in str(step.get("run", "")): + return step + return None + + +# ── rules: pure functions, each returning the violations it found ──────────── + + +def _g1_declared_checkouts_are_present_and_pointed_at( + workflow: dict, siblings: dict[str, Sibling] +) -> list[str]: + """Every ``ci_checkout=True`` sibling is checked out AND its env var points at it.""" + by_name = {_repo_name(s): s for s in _sibling_steps(workflow)} + pytest_step = _pytest_step(workflow) + env = (pytest_step or {}).get("env") or {} + + problems = [] + for name, sibling in siblings.items(): + if not sibling.ci_checkout: + continue + step = by_name.get(name) + if step is None: + problems.append(f"{name}: declared ci_checkout=True but no checkout step") + continue + expected = "${{ github.workspace }}/" + str(step["with"].get("path", "")) + actual = env.get(sibling.env) + if actual is None: + problems.append( + f"{name}: checked out, but {sibling.env} is not set on the step that " + "runs pytest — the tests would look in the conventional location, find " + "nothing, and skip" + ) + elif str(actual).strip() != expected: + problems.append( + f"{name}: {sibling.env}={actual!r} does not match the checkout path " + f"({expected!r})" + ) + return problems + + +def _g2_every_checkout_is_declared( + workflow: dict, siblings: dict[str, Sibling] +) -> list[str]: + """No sibling is fetched in CI without being declared with ``ci_checkout=True``.""" + problems = [] + for step in _sibling_steps(workflow): + name = _repo_name(step) + sibling = siblings.get(name) + if sibling is None: + problems.append( + f"{name}: checked out in CI but absent from SIBLINGS. A sibling nobody " + "declared is one no rule below covers." + ) + elif not sibling.ci_checkout: + problems.append( + f"{name}: checked out in CI but declared ci_checkout=False. One of the " + "two is wrong, and the declaration is what the reader believes." + ) + return problems + + +def _g4_every_note_names_a_record( + workflow: dict, siblings: dict[str, Sibling] +) -> list[str]: + """A sibling NOT checked out must say why, and **every** note must name a record. + + A deferral needs a trigger and an owner (ADR-014 §4). Requiring the note to cite a + `C-nn` or an `ADR-nnn` is what makes the citation non-optional, so declared + non-coverage stays attached to something a reader can follow. + + **Widened 2026-08-10 after external review.** The rule used to look only at siblings + with ``ci_checkout=False``, which left a hole in exactly the place ADR-016 calls out: + `views-crafdapi` was fetched *temporarily*, until the check it served moved to a + registry read, and that fact lived in a note **no rule examined**. (That fetch is gone + as of #248; the rule this incident produced is not.) A claim about + another repository, in prose, that nothing can check — which is the failure ADR-016 + §3 diagnoses, reintroduced for the one download the document itself calls temporary. + + So the note requirement now follows the *note*, not the flag: whenever a sibling + carries one, it must cite the record that owns it, whether it explains an exclusion + or a temporary inclusion. Siblings with no note and `ci_checkout=True` are unaffected + — a permanent, unremarkable fetch needs no justification. + """ + problems = [] + for name, sibling in siblings.items(): + if not sibling.ci_checkout and not sibling.note.strip(): + problems.append( + f"{name}: not checked out in CI and no note says why. Silent " + "non-coverage reads as 'nothing to see here'." + ) + continue + if sibling.note.strip() and not any( + token in sibling.note for token in ("C-", "ADR-", "#") + ): + problems.append( + f"{name}: its note explains something but names no record. Cite the " + "entry, ADR or issue that owns it — a note nothing points at is the " + "prose this file exists to replace." + ) + return problems + + +def _g5_no_step_swallows_its_own_failure( + workflow: dict, siblings: dict[str, Sibling] +) -> list[str]: + """No sibling checkout may carry ``continue-on-error``. + + One key and a failed fetch stops failing the build: the checks that sibling enables + go back to skipping, silently, on a green run. That is the state this whole file + exists to leave behind, and it is exactly the key someone reaches for while debugging + a red CI. + """ + return [ + f"{_repo_name(step)}: sibling checkout carries continue-on-error, so a failed " + "fetch would not fail the build — and the checks it enables would skip silently" + for step in _sibling_steps(workflow) + if step.get("continue-on-error") + ] + + +def _g6_siblings_land_under_the_excluded_path( + workflow: dict, siblings: dict[str, Sibling] +) -> list[str]: + """Sibling checkouts go under ``_siblings/`` — a real coupling to the ruff config.""" + return [ + f"{_repo_name(step)}: checked out to {step['with'].get('path')!r}, outside " + f"{_SIBLING_PATH_PREFIX!r}. pyproject.toml excludes only that prefix from ruff, " + "so the Lint step would start reporting another repository's code (it once " + "reported 745 findings that way)." + for step in _sibling_steps(workflow) + if not str(step["with"].get("path", "")).startswith(_SIBLING_PATH_PREFIX) + ] + + +def _g7_siblings_are_taken_from_main( + workflow: dict, siblings: dict[str, Sibling] +) -> list[str]: + """Every sibling checkout declares ``ref: main``. + + Without it `actions/checkout` takes the sibling's own default branch — which for + views-appwrite is `development`. ADR-014 §3 makes `main` the authority for a claim + about another repository, and `test_the_pinned_commit_is_reachable_from_the_contract_repos_main` + already enforces that. Reading the working tree from `development` while demanding + reachability from `main` is two guards in one file asking for different things. + """ + return [ + f"{_repo_name(step)}: checkout does not declare `ref: main`, so it takes that " + "repository's default branch — someone else's setting, and not the branch this " + "repo treats as authoritative." + for step in _sibling_steps(workflow) + if str(step["with"].get("ref", "")) != "main" + ] + + +_RULES = { + "G1 declared checkouts are present and pointed at": _g1_declared_checkouts_are_present_and_pointed_at, + "G2 every checkout is declared": _g2_every_checkout_is_declared, + "G4 every note names a record": _g4_every_note_names_a_record, + "G5 no step swallows its own failure": _g5_no_step_swallows_its_own_failure, + "G6 siblings land under the excluded path": _g6_siblings_land_under_the_excluded_path, + "G7 siblings are taken from main": _g7_siblings_are_taken_from_main, +} + + +# ── the inputs are real (ADR-014 §2) ───────────────────────────────────────── + + +def test_the_workflow_this_file_scans_actually_parses(): + """A YAML restructure would empty every scan below and report success. + + This is by far the most likely way this file becomes decoration: rename the job, nest + `steps` differently, move the workflow, and every rule below finds nothing to complain + about. So assert there is something to scan before trusting that there was nothing + wrong. + """ + assert _TEST_WORKFLOW.exists(), f"no workflow at {_TEST_WORKFLOW}" + workflow = _load(_TEST_WORKFLOW) + assert _steps(workflow), ( + "no steps parsed out of run_pytest.yml — the job or step structure moved, and " + "every rule in this file is now scanning an empty list." + ) + assert _pytest_step(workflow) is not None, ( + "no step in run_pytest.yml runs pytest. G1 checks the env vars on that step; " + "without it, it checks nothing." + ) + found = {_repo_name(s) for s in _sibling_steps(workflow)} + assert found, ( + "no sibling checkout steps found. Either they were all removed — in which case " + "the cross-repo checks are dark again — or the detection predicate no longer " + "matches how they are written." + ) + + +def test_every_workflow_file_is_scanned_for_smuggled_siblings(): + """G2 must see ALL workflows, not just this one. + + A sibling checkout added to `codeql.yml` would be invisible to a rule scoped to + `run_pytest.yml`. Nothing forbids one being added there; this makes it visible. + """ + workflows = sorted(_WORKFLOWS.glob("*.yml")) + assert workflows, ( + f"no workflow files under {_WORKFLOWS} — a glob over a moved directory finds " + "nothing and reports success (register C-74)." + ) + stray = [] + for path in workflows: + stray += [ + f"{path.name}: {_repo_name(step)}" + for step in _sibling_steps(_load(path)) + if _repo_name(step) not in SIBLINGS + ] + assert not stray, ( + f"sibling checkouts in workflow files that SIBLINGS does not declare: {stray}." + ) + + +def test_the_declared_siblings_are_the_ones_the_suite_resolves(): + """`SIBLINGS` is what every gated check resolves through; assert it is not empty.""" + assert SIBLINGS, "SIBLINGS is empty — every cross-repo check now skips permanently." + assert all(s.env.startswith("VIEWS_") for s in SIBLINGS.values()), ( + "a declared override variable does not follow the VIEWS_* convention; " + f"{ {n: s.env for n, s in SIBLINGS.items()} }" + ) + + +# ── the rules, against the real workflow ───────────────────────────────────── + + +@pytest.mark.parametrize("label", sorted(_RULES)) +def test_the_workflow_agrees_with_the_declaration(label): + """Each rule, run against the file that actually ships.""" + violations = _RULES[label](_load(_TEST_WORKFLOW), SIBLINGS) + assert not violations, f"[{label}]\n " + "\n ".join(violations) + + +# ── the rules, against synthetic mutants (ADR-014 §2) ──────────────────────── +# +# A rule that has only ever been run against a correct file has never been watched fail. +# Each case below is a minimal broken world; the rule must object AND name the sibling, +# because a violation nobody can locate is barely better than none. + + +def _workflow(**step_overrides) -> dict: + """A minimal well-formed workflow with one sibling checkout, then mutated.""" + with_block = { + "repository": "views-platform/views-appwrite", + "ref": "main", + "path": "_siblings/views-appwrite", + } + with_block.update(step_overrides.pop("with", {})) + step = {"uses": "actions/checkout@v3", "with": with_block} + step.update(step_overrides) + return { + "jobs": { + "test": { + "steps": [ + {"uses": "actions/checkout@v3"}, # this repo's own — no `repository:` + step, + { + "run": "poetry run pytest tests/", + "env": { + "VIEWS_APPWRITE": "${{ github.workspace }}/_siblings/views-appwrite" + }, + }, + ] + } + } + } + + +_ONE = {"views-appwrite": SIBLINGS["views-appwrite"]} + + +def _replace(name: str, **changes) -> dict[str, Sibling]: + from dataclasses import replace + + return {name: replace(SIBLINGS[name], **changes)} + + +#: ``(rule, broken workflow, broken declaration, the sibling the message must name, why)`` +#: +#: **The offender is declared, not inferred.** It was briefly derived from the mutant's +#: own sibling map — which reads as principled and is wrong for `G2`, whose whole premise +#: is an EMPTY declaration: there is nothing in the map to derive from, so it fell back to +#: a hardcoded ``"views-appwrite"`` while the assertion message claimed it had not. For G2 +#: the offender is a name in the *workflow*, and no amount of looking at the declaration +#: will find it. Writing it out is this repository's own rule (ADR-003): declare the fact, +#: do not let a helper guess it. +_MUTANTS = [ + ("G1 declared checkouts are present and pointed at", + {"jobs": {"test": {"steps": [{"run": "poetry run pytest tests/", "env": {}}]}}}, + _ONE, "views-appwrite", "declared ci_checkout=True but the checkout step is gone"), + ("G2 every checkout is declared", + _workflow(), {}, "views-appwrite", "checked out but undeclared"), + ("G4 every note names a record", + _workflow(), _replace("views-appwrite", ci_checkout=False, note=""), + "views-appwrite", "not checked out and no note"), + ("G4 every note names a record", + _workflow(), _replace("views-appwrite", ci_checkout=False, note="because reasons"), + "views-appwrite", "note explains but names no record"), + ("G5 no step swallows its own failure", + _workflow(**{"continue-on-error": True}), _ONE, + "views-appwrite", "a failed fetch would not fail the build"), + ("G6 siblings land under the excluded path", + _workflow(**{"with": {"path": "vendor/views-appwrite"}}), _ONE, + "views-appwrite", "outside _siblings/, so ruff would lint it"), + ("G7 siblings are taken from main", + _workflow(**{"with": {"ref": "development"}}), _ONE, + "views-appwrite", "takes a branch that is not the authority"), +] + + +def test_every_rule_has_a_mutant_that_proves_it_bites(): + """ADR-014 §2, applied to this file's own contents. + + Adding a rule to `_RULES` without a broken world to try it against is adding a rule + nobody has watched fail. That is the definition of decoration, and it is easy to do + by accident — the rule reads fine, the real workflow satisfies it, and the suite goes + green forever. + + This is a cheap structural check, not a clever one: it cannot tell whether a mutant + is a *good* one. It only refuses the case where there is none at all. + """ + proven = {label for label, *_ in _MUTANTS} + unproven = sorted(set(_RULES) - proven) + assert not unproven, ( + f"rules with no mutant in _MUTANTS: {unproven}. Add a broken world that each one " + "must object to, or the rule is untested against anything but a correct file." + ) + + +@pytest.mark.parametrize("label, workflow, siblings, offender, why", _MUTANTS) +def test_each_rule_bites_on_a_broken_world(label, workflow, siblings, offender, why): + """Feed a rule a deliberately broken world and demand that it complains. + + **A failure here means the RULE is broken, not the workflow.** That inversion is the + one thing about this file worth knowing before you debug it: `run_pytest.yml` is not + involved, the world being fed in is a synthetic dict defined below, and a green real + workflow is entirely consistent with this failing. A guard that no longer objects to + a violation is a guard that has silently stopped working, which is the failure this + whole file exists to make loud (ADR-014 §2). + + The second assertion is not pedantry: a rule that objects without naming which + sibling is at fault leaves a maintainer grepping a workflow file by hand. + """ + violations = _RULES[label](workflow, siblings) + assert violations, ( + f"[{label}] did not object to a world that is deliberately broken: {why}. " + "This means the RULE has stopped working — the real workflow is not involved " + "and is probably fine. Fix the rule, not run_pytest.yml." + ) + assert any(offender in v for v in violations), ( + f"[{label}] objected, but its message does not name the offending sibling " + f"({offender!r}): {violations}. A maintainer reading only the failure would not " + "know which repository to look at.\n\n" + "Each mutant declares the name its rule must produce, in _MUTANTS. If you have " + "changed which sibling a mutant breaks, change its declared offender in the same " + "edit — under this file's inverted semantics, a stale one fails a rule that works." + ) + + +@pytest.mark.parametrize( + "env, why", + [ + ({}, "the env var is not set at all"), + ({"VIEWS_APPWRITE": ""}, "set to empty — the CI misconfiguration that skips silently"), + ({"VIEWS_APPWRITE": "${{ github.workspace }}/_siblings/appwrite"}, "points at the wrong path"), + ], +) +def test_g1_bites_on_every_way_the_env_var_can_be_wrong(env, why): + """The three shapes of "checked out, but the tests cannot see it".""" + workflow = _workflow() + workflow["jobs"]["test"]["steps"][2]["env"] = env + violations = _g1_declared_checkouts_are_present_and_pointed_at(workflow, _ONE) + assert violations, f"G1 did not object to: {why}" + + +def test_g1_looks_at_the_step_that_runs_pytest_and_not_merely_any_step(): + """Env on the Lint step satisfies "the variable is set somewhere" and nothing else.""" + workflow = _workflow() + workflow["jobs"]["test"]["steps"][2]["env"] = {} + workflow["jobs"]["test"]["steps"].insert( + 2, {"run": "poetry run ruff check .", + "env": {"VIEWS_APPWRITE": "${{ github.workspace }}/_siblings/views-appwrite"}} + ) + assert _g1_declared_checkouts_are_present_and_pointed_at(workflow, _ONE), ( + "G1 accepted an env var set on the Lint step — the tests would never see it" + ) + + +def test_the_repos_own_checkout_is_not_mistaken_for_a_sibling(): + """G2 must not fire on line one of every workflow ever written (ADR-014 §3).""" + own_checkout_only = {"jobs": {"test": {"steps": [{"uses": "actions/checkout@v3"}]}}} + assert not _g2_every_checkout_is_declared(own_checkout_only, SIBLINGS) + assert not _sibling_steps(own_checkout_only) diff --git a/tests/test_clone_readiness.py b/tests/test_clone_readiness.py index 549bb77..79413fe 100644 --- a/tests/test_clone_readiness.py +++ b/tests/test_clone_readiness.py @@ -64,7 +64,6 @@ "views_postprocessing.contract.historical", "views_postprocessing.contract.gaul_lookup", "views_postprocessing.contract.gaul_schema", - "views_postprocessing.contract.enrichment", "views_postprocessing.contract.launch_config", "views_postprocessing.contract.source_metadata", "views_postprocessing.contract.store_metadata", diff --git a/tests/test_doc_accuracy.py b/tests/test_doc_accuracy.py index 3e295d0..1c1c87d 100644 --- a/tests/test_doc_accuracy.py +++ b/tests/test_doc_accuracy.py @@ -54,6 +54,13 @@ # #150 / #151 / #153 — modules retired or moved out of `unfao/` by epic #148 r"delivery/identity", r"unfao/extraction", r"unfao/frames", r"unfao/historical", r"unfao/gaul_schema", r"unfao/wire", + # #90 / C-75 — the enricher retired once it was shown to have no production + # caller. Added 2026-08-05, and late: the deletion PR did not extend this list, + # which is exactly what the paragraph above says a deletion PR has not finished + # doing. Found by the mutation test for the new CIC-collaborator guard, which + # could NOT catch a reintroduced mention — that guard only sees classes that + # still exist, and this one no longer does. + r"GaulLookupEnricher", r"enrich_dataframe_with_pg_info", r"contract/enrichment", # #149 — pipeline-core's pandas container, no longer referenced by this repo. # Negative lookbehind: `FAO_PGMDataset` is views-faoapi's class and is live. # It inherits IGNORECASE, so `fao_PGMDataset` is spared too — no such spelling @@ -93,6 +100,185 @@ def _deleted_symbol_offenders(docs: list[Path]) -> list[str]: return offenders +#: A document may declare, once at the top, that its SUBJECT is a retirement — after +#: which the deleted-symbol scan skips it wholesale. +#: +#: ADR-011 is the case this exists for: it is the decision to *replace* the runtime +#: mapper, so `PriogridCountryMapper`, `geopandas` and `mapping.py` appear fourteen +#: times in it, every one correctly. Marking fourteen lines would be noise pretending +#: to be rigour; marking the document once is the same claim, made where a reader sees +#: it. Per-line ``legacy-ok`` remains the tool for an isolated mention in a document +#: that is otherwise about the present. +_RETIREMENT_DOC_MARKER = "` if its subject IS the retirement. " + "If it is not history, it is a stale path and the document is wrong." + ) + + +def test_the_retirement_marker_is_not_used_to_silence_a_live_document(): + """An escape hatch nobody bounds is just a disabled check. + + A whole-document exemption is the strongest thing available here, so the set of + documents holding one is pinned. Adding another is a deliberate act that shows up in + a diff, rather than a quiet way to stop a guard complaining. + """ + marked = sorted( + d.name for d in _governance_docs() if _RETIREMENT_DOC_MARKER in d.read_text() + ) + assert marked == ["011_replace_runtime_mapper_with_precomputed_lookup.md"], ( + f"documents exempted from the deleted-symbol scan: {marked}. Only a document " + "whose SUBJECT is a retirement earns this. If a present-tense document needs " + "it, the document is describing code that no longer exists." + ) + + def test_living_docs_have_no_deleted_symbol_references(): offenders = _deleted_symbol_offenders(_living_docs()) assert not offenders, "deleted-symbol references in living docs:\n" + "\n".join(offenders) @@ -181,6 +367,10 @@ def test_internal_doc_links_resolve(): #: previously unbudgeted, which is the same regrowth wearing a different filename. _MANAGER_LINE_BUDGET = 450 +#: The manager CLASS, separately (C-40). 351 before the 2026-08-05 extraction, 272 after. +#: A ratchet — see `test_the_manager_class_itself_stays_thin` for why it is not a target. +_MANAGER_CLASS_BUDGET = 300 + def _is_type_checking(test: ast.expr) -> bool: """`TYPE_CHECKING` or `typing.TYPE_CHECKING`, and nothing else. @@ -253,9 +443,11 @@ def test_pandas_is_not_imported_at_runtime_anywhere_in_the_package(): "the representation the migration removed. If it is genuinely needed, put it " "behind `if TYPE_CHECKING` or say in ADR-012 why it is not." ) - assert type_only == ["contract/enrichment.py"], ( - f"the type-only pandas imports moved: {type_only}. Not necessarily wrong — but " - "ADR-012 names the seam, so update it rather than letting the claim drift." + assert type_only == [], ( + f"a type-only pandas import reappeared: {type_only}. There is no pandas in this " + "package at all since #90 retired contract/enrichment.py (register C-75) — epic " + "#85's claim is now literal, not nearly-true. If one is genuinely needed, say in " + "ADR-012 why before adding it back." ) @@ -300,6 +492,37 @@ def test_the_manager_stays_within_its_line_budget(managers_dir): ) +@pytest.mark.parametrize("partner", _PARTNER_PACKAGES) +def test_the_manager_class_itself_stays_thin(partner): + """The directory budget above is anti-regrowth. This one is anti-*fusion*. + + They measure different things and both are needed. A file can sit comfortably under + 450 lines while the manager **class** absorbs store construction, env assembly and + business logic — which is exactly what register C-40 is about, and exactly what the + directory count cannot see. Conversely a thin class proves nothing if 800 lines + moved to a sibling module, which is why the directory count exists. + + Added 2026-08-05 alongside the C-40 extraction that took the class from 351 lines to + 272. The bound is a **ratchet, not a target**: it is the measured value plus modest + headroom, and the right response to it failing is to move something out of the class, + not to raise the number. If it is ever raised, the raise belongs in a commit message + that says what was added and why it had to live on the manager. + """ + source = (_PKG / partner / "managers" / f"{partner}.py").read_text() + cls = next( + n for n in ast.walk(ast.parse(source)) + if isinstance(n, ast.ClassDef) and n.name.endswith("PostProcessorManager") + ) + lines = cls.end_lineno - cls.lineno + 1 + assert lines <= _MANAGER_CLASS_BUDGET, ( + f"{cls.name} is {lines} lines, over the {_MANAGER_CLASS_BUDGET} ratchet. The " + "manager is the Template-Method shell: it orchestrates hooks and adapts the " + "framework. Logic that can be called with declared arguments belongs in a " + "module-level function or in contract/ — where it can be tested without a " + "manager, an Appwrite environment or a views-models path manager (C-40 (a))." + ) + + def _imported_subpackages(source: str, module_path: Path) -> set[str]: """Every ``views_postprocessing.`` this module imports — all four spellings. @@ -448,7 +671,7 @@ def test_the_ban_covers_the_post_148_deletions_and_spares_the_live_lookalike(): ``_BANNED`` says a deletion PR that does not extend it has not finished. No test can substitute for that; only the habit can. 2. **Too broad** — `FAO_PGMDataset` is a *views-faoapi* class, alive and correctly - cited in `contract/gaul_schema.py` and `contract/enrichment.py`. A careless + cited in `contract/gaul_schema.py`. A careless `PGMDataset` pattern would ban a true statement about another repo's code. """ for deleted in ( diff --git a/tests/test_enrichment.py b/tests/test_enrichment.py deleted file mode 100644 index ca1ea6c..0000000 --- a/tests/test_enrichment.py +++ /dev/null @@ -1,419 +0,0 @@ -"""Tests for the lookup-based enricher (ADR-011, Stage 1). - -Run against the committed gaul_lookup.parquet — no datafactory checkout or -shapefiles required. These tests pin the contract the enricher must satisfy so -that swapping it in for the runtime mapper (Stage 3) is invisible downstream. -""" - -import pandas as pd -import logging - -import numpy as np -import pyarrow as pa -import pyarrow.parquet as pq -import pytest - -from views_postprocessing.contract import gaul_lookup -from views_postprocessing.contract.enrichment import ( - GaulLookupEnricher, - METADATA_COLS, -) - -# The 5 africa_me_legacy ocean cells that have no GAUL assignment — they must -# NOT be in the lookup (they are excluded upstream by the land_gaul region). -OCEAN_CELLS = [62356, 94776, 99027, 107733, 107742] - -CODE_COLS = ["admin1_gaul1_code", "admin1_gaul0_code", "admin2_gaul2_code"] -NAME_COLS = ["admin1_gaul1_name", "admin1_gaul0_name", "admin2_gaul2_name", - "country_iso_a3"] - - -@pytest.fixture(scope="module") -def enricher(): - return GaulLookupEnricher() - - -@pytest.fixture(scope="module") -def lookup(): - """The committed artifact as a pandas frame, built here rather than reached for. - - Until S4 (#89) this was ``enricher._lookup`` — the enricher's own private pandas - frame. It no longer has one: the lookup side is numpy + pyarrow, and pandas is a - type-only import there. These tests are about the **artifact**, not the enricher, - so they load it directly. - - The ``set_index`` is deliberate and forward-looking. ``to_pandas()`` currently - restores ``priogrid_gid`` as the index from the parquet's embedded pandas - metadata — metadata that **S5 (#90) removes** when the builder becomes - pyarrow-native. Handling both shapes means these tests do not have to change - again then. - """ - df = gaul_lookup.load().to_pandas() - if "priogrid_gid" in df.columns: - df = df.set_index("priogrid_gid") - return df - - -class TestLookupIntegrity: - """The committed lookup must be clean by construction.""" - - def test_expected_cell_count(self, lookup): - # land_gaul region: 64,742 fully-complete land cells. - assert len(lookup) == 64_742 - - def test_no_nulls_anywhere(self, lookup): - assert int(lookup.isna().sum().sum()) == 0 - - def test_no_minus_one_sentinel_in_codes(self, lookup): - for c in CODE_COLS: - assert (lookup[c] != -1).all(), f"{c} has -1 sentinel" - - def test_no_empty_strings_in_names(self, lookup): - for c in NAME_COLS: - assert (lookup[c].astype(str).str.len() > 0).all() - - def test_has_exactly_the_nine_contract_columns(self, lookup): - assert list(lookup.columns) == METADATA_COLS - - def test_dtypes(self, lookup): - for c in CODE_COLS: - assert pd.api.types.is_numeric_dtype(lookup[c]), c - for c in ["pg_xcoord", "pg_ycoord"]: - assert pd.api.types.is_float_dtype(lookup[c]), c - for c in NAME_COLS: - assert isinstance(lookup[c].dtype, pd.CategoricalDtype), c - - def test_ocean_cells_excluded(self, lookup): - for gid in OCEAN_CELLS: - assert gid not in lookup.index, f"ocean cell {gid} should be excluded" - - -class TestCoordinateFormula: - """pg_xcoord/pg_ycoord must follow the PRIO-GRID 0.5-degree formula.""" - - def test_known_coordinates(self, lookup): - for gid in lookup.index[:50]: - x = -180.0 + ((gid - 1) % 720) * 0.5 + 0.25 - y = -90.0 + ((gid - 1) // 720) * 0.5 + 0.25 - assert lookup.loc[gid, "pg_xcoord"] == pytest.approx(x) - assert lookup.loc[gid, "pg_ycoord"] == pytest.approx(y) - - -class TestEnricherContract: - """enrich() must reproduce the 9-column contract the manager consumes.""" - - def _frame(self, gids): - return pd.DataFrame({ - "priogrid_gid": gids, - "month_id": [100] * len(gids), - }) - - def test_all_metadata_cols_present(self, enricher, lookup): - gids = lookup.index[:10].tolist() - out = enricher.enrich_dataframe_with_pg_info( - self._frame(gids), pg_id_col="priogrid_gid", time_id_col="month_id", - ) - for c in METADATA_COLS: - assert c in out.columns - - def test_no_nulls_for_known_gids(self, enricher, lookup): - gids = lookup.index[:100].tolist() - out = enricher.enrich_dataframe_with_pg_info( - self._frame(gids), pg_id_col="priogrid_gid", time_id_col="month_id", - ) - for c in METADATA_COLS: - assert out[c].isna().sum() == 0, c - - def test_values_match_lookup(self, enricher, lookup): - gids = lookup.index[:20].tolist() - out = enricher.enrich_dataframe_with_pg_info( - self._frame(gids), pg_id_col="priogrid_gid", time_id_col="month_id", - ).set_index("priogrid_gid") - for gid in gids: - assert out.loc[gid, "country_iso_a3"] == lookup.loc[gid, "country_iso_a3"] - assert out.loc[gid, "admin2_gaul2_code"] == lookup.loc[gid, "admin2_gaul2_code"] - - def test_row_count_preserved(self, enricher, lookup): - gids = lookup.index[:7].tolist() - out = enricher.enrich_dataframe_with_pg_info( - self._frame(gids), pg_id_col="priogrid_gid", time_id_col="month_id", - ) - assert len(out) == 7 - - def test_codes_numeric_coords_float(self, enricher, lookup): - gids = lookup.index[:5].tolist() - out = enricher.enrich_dataframe_with_pg_info( - self._frame(gids), pg_id_col="priogrid_gid", time_id_col="month_id", - ) - for c in CODE_COLS: - assert pd.api.types.is_numeric_dtype(out[c]), c - for c in ["pg_xcoord", "pg_ycoord"]: - assert pd.api.types.is_float_dtype(out[c]), c - - -class TestFailLoud: - """Unknown / excluded cells must surface as NaN, not a sentinel.""" - - def test_unknown_gid_yields_null(self, enricher, lookup): - good = int(lookup.index[0]) - df = pd.DataFrame({"priogrid_gid": [good, 999_999], "month_id": [1, 1]}) - out = enricher.enrich_dataframe_with_pg_info( - df, pg_id_col="priogrid_gid", time_id_col="month_id", - ).set_index("priogrid_gid") - assert out.loc[good, "country_iso_a3"] is not None - assert pd.isna(out.loc[999_999, "country_iso_a3"]) - for c in CODE_COLS: - assert pd.isna(out.loc[999_999, c]) - - def test_ocean_cell_yields_null(self, enricher): - df = pd.DataFrame({"priogrid_gid": OCEAN_CELLS, - "month_id": [1] * len(OCEAN_CELLS)}) - out = enricher.enrich_dataframe_with_pg_info( - df, pg_id_col="priogrid_gid", time_id_col="month_id", - ) - # Every ocean cell is absent from the lookup -> all-null metadata. - assert out["country_iso_a3"].isna().all() - - def test_missing_pg_id_col_raises(self, enricher): - with pytest.raises(ValueError, match="not found"): - enricher.enrich_dataframe_with_pg_info( - pd.DataFrame({"x": [1]}), pg_id_col="priogrid_gid", - ) - - -# ── the guarantees the CIC states, pinned (S4 / #89; ADR-014 §1) ───────────── -# -# The CIC claims row/order/index preservation "verified across eight input shapes" -# and names the output dtypes. That verification was a throwaway script run once -# during development — a guarantee resting on a claim rather than on a check, which -# is precisely what ADR-014 §1 forbids, written by the story after the ADR landed. -# These commit it. - - -class TestConstructionRefusals: - """Degenerate lookups must fail at construction, naming the lookup.""" - - def _write(self, path, gids): - cols = {"priogrid_gid": pa.array(gids, pa.int64())} - for c in METADATA_COLS: - cols[c] = pa.array( - [0.0] * len(gids) if c in ("pg_xcoord", "pg_ycoord") or c.endswith("_code") - else ["x"] * len(gids), - pa.float64() if c in ("pg_xcoord", "pg_ycoord") or c.endswith("_code") else pa.string(), - ) - t = pa.table(cols).replace_schema_metadata({b"lookup_version": b"t@00000000"}) - pq.write_table(t, path) - return path - - def test_an_empty_lookup_is_refused_at_construction(self, tmp_path): - """It used to raise IndexError from inside the gather instead — the guard - `(len(self._keys) > 0) & (...)` read as a guard and was not one, because `&` - evaluates both operands.""" - path = self._write(tmp_path / "empty.parquet", []) - with pytest.raises(ValueError, match="is empty"): - GaulLookupEnricher(path) - - def test_a_null_key_is_refused_at_construction(self, tmp_path): - """A null key would coerce to the same sentinel as an unusable query id, the - two would collide, and the row would be reported FOUND — receiving another - cell's metadata. That is the fabricated value this module forbids (cf. C-35).""" - cols = {"priogrid_gid": pa.array([1, None, 3], pa.int64())} - for c in METADATA_COLS: - numeric = c in ("pg_xcoord", "pg_ycoord") or c.endswith("_code") - cols[c] = pa.array([0.0] * 3 if numeric else ["x"] * 3, - pa.float64() if numeric else pa.string()) - path = tmp_path / "nullkey.parquet" - pq.write_table(pa.table(cols).replace_schema_metadata({b"lookup_version": b"t@0"}), path) - with pytest.raises(ValueError, match="null values"): - GaulLookupEnricher(path) - - -class TestUnusableCellIds: - """A value that is not a cell id gathers to null — never to a guessed cell. - - Each case was verified against the pandas merge this replaced; the behaviour - below is that merge's, not an invention. The drifted-float case is the one that - matters most: a blind `astype(np.int64)` truncates `54220.000000001` to `54220` - and matches a real, *different* cell, silently. - """ - - def _out(self, enricher, col): - return enricher.enrich_dataframe_with_pg_info( - pd.DataFrame({"priogrid_gid": col, "month_id": [1] * len(col)}), - pg_id_col="priogrid_gid", time_id_col="month_id", - ) - - def test_a_missing_gid_yields_null_and_does_not_crash(self, enricher, lookup): - good = lookup.index[:2].tolist() - out = self._out(enricher, [float(good[0]), float("nan"), float(good[1])]) - assert len(out) == 3 - assert out["country_iso_a3"].isna().sum() == 1 - - def test_a_non_integral_gid_never_matches_a_neighbouring_cell(self, enricher, lookup): - gid = int(lookup.index[100]) - out = self._out(enricher, [float(gid) + 1e-9]) - assert out["country_iso_a3"].isna().all(), ( - "a drifted float matched a cell — truncation turned it into a different, " - "real gid and fabricated that cell's geography" - ) - - def test_pandas_na_yields_null_rather_than_a_bare_typeerror(self, enricher, lookup): - good = lookup.index[:2].tolist() - out = self._out(enricher, pd.array([good[0], pd.NA, good[1]], dtype="Int64")) - assert len(out) == 3 and out["country_iso_a3"].isna().sum() == 1 - - def test_an_unusable_id_cannot_match_the_cell_it_was_substituted_with(self, tmp_path): - """The `& usable` term in `_gather`, proven load-bearing (ADR-014 §2). - - It survived removal against the committed artifact: an unusable id converts to - `0`, and `0` is not a real gid, so the key comparison already failed. That made - it look like decoration. It is not — it is the only thing standing between an - unusable id and the row it was substituted onto, and the substitute is a real - key the moment a lookup contains gid `0`. - - Without the mask, the NaN row below silently receives cell 0's geography. That - is the fabricated value this module's docstring forbids, produced from an input - the caller never wrote. - """ - cols = {"priogrid_gid": pa.array([0, 5, 9], pa.int64())} - for c in METADATA_COLS: - numeric = c in ("pg_xcoord", "pg_ycoord") or c.endswith("_code") - cols[c] = pa.array( - [0.0] * 3 if numeric else ["ZERO", "FIVE", "NINE"], - pa.float64() if numeric else pa.string(), - ) - path = tmp_path / "withzero.parquet" - pq.write_table( - pa.table(cols).replace_schema_metadata({b"lookup_version": b"t@00000000"}), path - ) - - out = GaulLookupEnricher(path).enrich_dataframe_with_pg_info( - pd.DataFrame({"priogrid_gid": [5.0, float("nan"), 9.0], "month_id": [1, 1, 1]}), - pg_id_col="priogrid_gid", time_id_col="month_id", - ) - assert out["country_iso_a3"].tolist() == ["FIVE", None, "NINE"], ( - "an unusable id matched the cell it was substituted with — `& usable` is " - "what prevents that, and this is the input that proves it" - ) - - def test_a_string_gid_column_is_refused_rather_than_parsed(self, enricher, lookup): - """A string gid is a declaration error, and the merge this replaced said so. - - `pandas` raised *"You are trying to merge on object and int64 columns. If you - wish to proceed you should use pd.concat"*. An - earlier draft of `_as_cell_ids` used `float(value)`, which happily parsed - `"54220"` and matched — inference where the old path declared (ADR-003). - """ - gid = str(int(lookup.index[0])) - out = self._out(enricher, [gid]) - assert out["country_iso_a3"].isna().all(), ( - f"the string {gid!r} was parsed into a cell id; a string column is a " - "declaration error, not a gid to be guessed at" - ) - - @pytest.mark.parametrize( - "label,array", - [ - # Each case must enter the branch it is testing. An earlier draft built - # them all with dtype=object, which routed every one through the - # element-wise branch — so the int and float bounds it claimed to test - # were never executed, and removing them left the suite green. - ("uint64 max (int branch)", np.array([np.iinfo(np.uint64).max], dtype=np.uint64)), - ("1e30 (float branch)", np.array([1e30], dtype=np.float64)), - ("inf (float branch)", np.array([np.inf], dtype=np.float64)), - ("-inf (float branch)", np.array([-np.inf], dtype=np.float64)), - ("string (object branch)", np.array(["123"], dtype=object)), - ("bool (object branch)", np.array([True], dtype=object)), - ("float exactly 2**63", np.array([2.0**63], dtype=np.float64)), - ("timedelta64", np.array([1], dtype="timedelta64[ns]")), - ], - ) - def test_out_of_range_and_non_numeric_values_are_unusable_not_wrapped( - self, enricher, label, array - ): - """Outside int64 the cast WRAPS rather than raising, and a wrapped id flagged - valid is exactly the fabricated value this conversion exists to prevent. - - `uint64` max becomes `-1`; `1e30` and `float(2**63)` become `INT64_MIN`. Each - passes a naive finite-and-integral test, so the bounds are separate conditions - rather than consequences of the others. - - Asserted through the PUBLIC method. An earlier draft called the private - `_as_cell_ids` and checked its mask — pinning an implementation detail rather - than the guarantee a caller relies on, and skipping `_gather`'s use of that - mask entirely. Every dtype below is reachable through a DataFrame column, so - there was never a reason to reach inside. - """ - out = enricher.enrich_dataframe_with_pg_info( - pd.DataFrame({"priogrid_gid": array, "month_id": [1] * len(array)}), - pg_id_col="priogrid_gid", time_id_col="month_id", - ) - assert out["country_iso_a3"].isna().all(), ( - f"{label} produced a match — an id the caller never wrote was treated as " - "a cell and that cell's geography was fabricated onto the row" - ) - - # AND at the mask, because the public assertion above cannot tell "refused" - # from "accepted but wrapped to an id that happens to be absent" — both give - # null. Reverting the float bound or re-admitting timedelta64 left the public - # assertion green while the value was silently accepted; only this sees it. - # The public check is the caller's guarantee; this one is the mutation proof. - _, usable = GaulLookupEnricher._as_cell_ids(array) - assert not usable.any(), ( - f"{label} was ACCEPTED as a cell id (it only looked refused because the " - f"wrapped id is absent from this lookup — against a lookup that contains " - f"it, this row would receive that cell's geography)" - ) - - def test_the_warning_names_unknown_cells_but_invents_no_id_for_unusable_ones( - self, enricher, lookup, caplog - ): - good = int(lookup.index[0]) - with caplog.at_level(logging.WARNING): - self._out(enricher, [float(good), float("nan"), 999999.0]) - message = " ".join(r.getMessage() for r in caplog.records) - assert "999999" in message, "the genuinely unknown cell must be named" - assert "no usable cell id" in message - assert "[0]" not in message, ( - "an unusable id was reported as cell 0 — a cell nobody asked about" - ) - - -class TestFramePropertiesPreserved: - """Row count, order and index survive the gather. Was CIC prose; now a check.""" - - @pytest.mark.parametrize("shape", ["ordered", "reversed", "duplicated", "single", "empty"]) - def test_row_count_and_order_are_preserved(self, enricher, lookup, shape): - gids = { - "ordered": lookup.index[:50].tolist(), - "reversed": lookup.index[:50].tolist()[::-1], - "duplicated": lookup.index[:5].tolist() * 3, - "single": lookup.index[:1].tolist(), - "empty": [], - }[shape] - out = enricher.enrich_dataframe_with_pg_info( - pd.DataFrame({"priogrid_gid": gids, "month_id": [1] * len(gids)}), - pg_id_col="priogrid_gid", time_id_col="month_id", - ) - assert len(out) == len(gids) - assert out["priogrid_gid"].tolist() == gids - - def test_a_non_default_index_is_preserved(self, enricher, lookup): - gids = lookup.index[:20].tolist() - df = pd.DataFrame({"priogrid_gid": gids, "month_id": [1] * 20}, - index=range(1000, 1020)) - out = enricher.enrich_dataframe_with_pg_info( - df, pg_id_col="priogrid_gid", time_id_col="month_id") - assert out.index.equals(df.index), ( - "the index moved. The pandas merge this replaced mutated it on empty " - "input; the gather must not mutate it on any input." - ) - - def test_name_columns_are_object_not_categorical(self, enricher, lookup): - """The CIC states this dtype. It changed in #89 and was measured, not guessed.""" - out = enricher.enrich_dataframe_with_pg_info( - pd.DataFrame({"priogrid_gid": lookup.index[:5].tolist(), "month_id": [1] * 5}), - pg_id_col="priogrid_gid", time_id_col="month_id", - ) - for c in NAME_COLS: - assert out[c].dtype == object, f"{c} is {out[c].dtype}, CIC says object" diff --git a/tests/test_env_declaration.py b/tests/test_env_declaration.py index 8be6bd1..5a59f7c 100644 --- a/tests/test_env_declaration.py +++ b/tests/test_env_declaration.py @@ -24,20 +24,29 @@ """ import ast +import hashlib import logging import re import subprocess +import sys from pathlib import Path import pytest +from tests.seam_registry import ( + ABSENT as _ABSENT, + REGISTRY_RELPATH as _REGISTRY_RELPATH, + RegistryReadError as _RegistryReadError, + registry_at as _registry_at, + registry_current as _registry_current, + rows as _rows, +) from tests.conftest import ( PARTNER_PACKAGES, broken_sibling_overrides, commit_is_on_main, git_output, require_sibling, - sibling_repo, ) from views_postprocessing.contract import launch_config from views_postprocessing.crafd import appwrite_env as crafd_env @@ -128,9 +137,24 @@ def _manager_source(partner: str) -> Path: _DOTENV_CALLS = {"load_dotenv", "find_dotenv", "dotenv_values"} -def _dotenv_use(source: str) -> list[str]: +def _parsed(path: Path) -> ast.Module: + """Parse ``path``, refusing a malformed module by path and line — never by content.""" + try: + return ast.parse(path.read_text()) + except SyntaxError as exc: + raise AssertionError(f"{path.name} does not parse (line {exc.lineno})") from None + + +def _dotenv_use(path: Path) -> list[str]: """Real imports of, and calls into, python-dotenv — parsed, not grepped. + A malformed module is refused by path and line, never by content. ``ast.parse`` + raises with the source as its own frame's argument, which pytest renders in full — so + a package file that both fails to parse and carries a coordinate value would publish + itself. ``from None`` drops the chained ``SyntaxError``, which carries the same text. + (An earlier version credited the ``Path`` parameter for this. Measured: it changes + nothing, because the text still reaches ``ast.parse``.) + **Prose is not a violation and must not be treated as one.** ``appwrite_env.py``'s own docstring explains what the retired borrow was, spelling it exactly; a token scan over the package flags that sentence and gets deleted, after which the rule it @@ -139,7 +163,7 @@ def _dotenv_use(source: str) -> list[str]: or a comment, so the guard can cover the whole package without lying about prose. """ found = [] - for node in ast.walk(ast.parse(source)): + for node in ast.walk(_parsed(path)): if isinstance(node, ast.Import): found += [a.name for a in node.names if a.name.split(".")[0] == "dotenv"] elif isinstance(node, ast.ImportFrom): @@ -186,7 +210,7 @@ def test_the_dotenv_borrow_is_dead(): offenders = { source.relative_to(_PKG).as_posix(): used for source in sorted(_PKG.rglob("*.py")) - for used in [_dotenv_use(source.read_text())] + for used in [_dotenv_use(source)] if used } assert not offenders, ( @@ -263,6 +287,66 @@ def test_declared_names_match_the_manager_reads(partner): assert text.count("appwrite_env.assert_env_declared(") == 2 +def test_a_queryset_that_failed_to_import_is_refused_as_that(caplog): + """Register C-83 — the refusal must name the real fault, not a plausible one. + + `get_queryset()` returns ``None`` for **any** exception while importing + ``config_queryset.py``. Feeding that to ``declared_data_format`` yields + ``'dataframe'`` (its documented default for a non-dict), and the format guard then + tells the operator to set ``data_format: 'feature_frame'`` — in a file that already + says exactly that. + + Reproduced both ways on 2026-08-03: in an environment without views-datafactory the + real ``un_fao`` queryset reported ``dataframe`` while declaring ``feature_frame``; + in a complete environment the same file reported ``feature_frame``. + """ + import logging + + with caplog.at_level(logging.ERROR): + with pytest.raises(launch_config.LaunchConfigError) as excinfo: + launch_config.assert_queryset_was_importable(None) + + message = str(excinfo.value) + assert "could not be imported" in message + assert "data_format" in message and "do not edit" in message.lower(), ( + "the refusal must actively steer the operator AWAY from the config file — " + "that is the whole point, since the old message steered them into it" + ) + assert any(r.levelno >= logging.ERROR for r in caplog.records), ( + "ADR-008: a structural refusal leaves a persistent record, not just a traceback" + ) + + +def test_an_importable_queryset_passes_the_readability_check(): + """The guard must not stand in front of a queryset that imported fine. + + Anything not-None passes here; whether it declares the right format is the *next* + check's question, and keeping them separate is the entire fix. + """ + launch_config.assert_queryset_was_importable({"data_format": "feature_frame"}) + launch_config.assert_queryset_was_importable({}) # wrong, but importable + + +def test_both_managers_check_importability_before_asking_what_was_declared(): + """Order matters: reversed, the confusing message wins again. + + Source-scan because the managers need Appwrite env and a views-models path manager + to instantiate — the standing pattern for manager-side facts. + """ + for partner in _PARTNERS: + source = _manager_source(partner).read_text() + assert source.count("get_queryset()") == 1, ( + f"[{partner}] the queryset must be read ONCE and the value reused; reading " + "it twice invites the two checks to disagree about what they saw" + ) + importable = source.index("assert_queryset_was_importable") + declared = source.index("assert_frame_native_historical") + assert importable < declared, ( + f"[{partner}] the importability check must come FIRST. Reversed, a queryset " + "that failed to import is still reported as one declaring 'dataframe'." + ) + + # ── ADR-008 across BOTH entry validators (S1 / #182, register C-71) ────────── # # `appwrite_env` asserts the launcher assembled the *environment*; `launch_config` @@ -372,7 +456,6 @@ def test_secret_env_names_follow_the_seam_contract_naming_rule(): # and copying one into a test is the same violation as copying it into code. These # compare names, declared classes, and the edition — nothing else. -_REGISTRY_RELPATH = Path("docs") / "ADRs" / "platform" / "coordinate_registry.toml" #: How this module treats each declared name, vs. the registry's own `class` field. #: @@ -416,29 +499,132 @@ def test_every_declared_name_is_classified_here(partner): ) -def _load_registry(repo: Path) -> dict: - tomllib = pytest.importorskip( - "tomllib", - reason="tomllib is stdlib from Python 3.11; pyproject declares >=3.11, so a " - "conforming environment has it. CI runs 3.11.", - ) - return tomllib.loads((repo / _REGISTRY_RELPATH).read_text()) +#: Every top-level table the registry may carry, and what this repository does with it. +#: +#: **Declared, and asserted against the live registry** — a table appearing upstream that +#: nobody here has classified is a change this repository has not looked at. That is not +#: hypothetical: `[contract.*]` arrived in v1.5.0 carrying a live obligation (the +#: ADR-017 delivery-label mirror), and the ONLY mechanism here that noticed was a +#: version-string comparison that fired for the wrong reason and was deleted with this +#: change. Four tables were being ignored silently at the time. +_TABLE_ROLE = { + "connection": "CONSUMED", # parsed by _declared_classes; coordinates we read + "target": "CONSUMED", # parsed by _declared_classes; coordinates we read + "secret": "CONSUMED", # parsed by _declared_classes; the operator's slots + "contract": "MIRRORED", # values live in our source by design (ADR-017 §5); + # checked by tests/test_product.py, not by _declared_classes + "excluded": "IGNORED", # names the registry records as deliberately NOT coordinates + "test_environment": "IGNORED", # a fact about the platform, not about this package + #: Arrived at registry v1.6.0, and it is views-appwrite#76 delivered — each edition + #: marked ``obliges_consumers = true|false``, so a consumer can tell a console + #: observation from a change it must act on. IGNORED only because nothing here reads + #: it YET: adopting it is the follow-up that lets the drift check below stop caring + #: about editions that oblige nobody. Caught by this very partition on its first + #: encounter, which is what the partition is for. + "edition": "IGNORED", + "meta": "METADATA", # the edition and its amendment log +} + +#: direction is deliberately unchecked. (An earlier version of this comment offered +#: v1.5.1's removal of `[unmodelled]` as the worked example of a silent case. That was +#: WRONG: `unmodelled` was never in this partition, so against v1.4.4 it would have +#: been a RED build demanding classification. The rule is right; the illustration +#: was not, and it had been repeated in three places.) +#: silent while v1.5.0 adding `[contract]` is a red build with something to do. +#: The only roles that mean anything. A typo in `_TABLE_ROLE` used to be silent, and it +#: silently narrowed a security scan: mistyping "CONSUMED" dropped `target` from the +#: no-copy check's sections, taking it from twelve values to two, with no test objecting. +_ROLES = ("CONSUMED", "MIRRORED", "IGNORED", "METADATA") + +#: Tables this package reads rows out of. **One source, three consumers** — the class +#: check, the drift projection and the no-copy scan all derive from here rather than each +#: keeping a copy of the tuple. They did keep copies, which made classifying a table as +#: CONSUMED add exactly zero coverage while satisfying the guard that demanded it — the +#: guard's own remediation advice producing register C-74's shape. +_CONSUMED_TABLES = tuple(n for n, role in _TABLE_ROLE.items() if role == "CONSUMED") + +_TABLES_WE_DEPEND_ON = tuple( + name for name, role in _TABLE_ROLE.items() if role in ("CONSUMED", "MIRRORED") +) + + +def _unclassified_tables(registry: dict) -> list[str]: + """Tables upstream that nobody here has classified. THE production predicate. + + Extracted so the proof can call it instead of re-typing it. The previous proof + re-typed the set arithmetic, so neutering this to ``return []`` left it green — a + detector for the detector that detected nothing. + """ + return sorted(set(registry) - set(_TABLE_ROLE)) + + +def _missing_dependencies(registry: dict) -> list[str]: + """Tables we depend on that are absent upstream. THE production predicate.""" + return sorted(n for n in _TABLES_WE_DEPEND_ON if n not in registry) + + +def _projection(registry: dict, names: set[str]) -> dict[str, tuple]: + """``name -> (section, class, value-or-absent)`` for the names we declare. + + The unit of comparison between two editions. Restricted to our own surface, so an + unrelated row moving is invisible here — that is the whole point, and it is what + stops an observation-only bump from reddening this repository (register C-86). + + ``value`` is included because **rotation is the change no other check can see**: a + bucket id whose value changes keeps its name and its class, so the names-and-classes + check passes while every delivery goes to the wrong place. The no-copy rule forbids + writing the expected values into this repository, so the pinned edition is the only + lawful place a baseline for that comparison can live. + """ + return {n: row for n, row in _rows(registry, _CONSUMED_TABLES).items() if n in names} + + +def _describe_changes(then: dict, now: dict, names: set[str]) -> dict[str, str]: + """What differs, said WITHOUT printing a coordinate value. + + This repository is public and its CI logs are world-readable. The first version of + this comparison interpolated the projection tuples straight into the assertion + message — so the single event this check exists to fire on, a rotated coordinate, + would have printed the old and new values into a public log. Longer than the + six-character threshold the no-copy check itself uses to define a leak, and the + outcome ``tests/test_redaction_guard.py`` exists to prevent. + + So: name the field that moved, and for a value give a short digest — enough to see + that two editions disagree and to match against a rotation you performed, never + enough to be the value. + """ + def digest(value) -> str: + if value is _ABSENT: + return "" + return f"" + + out = {} + for name in sorted(names): + was, is_ = then.get(name), now.get(name) + if was == is_: + continue + if was is None or is_ is None: + out[name] = "absent at the pin" if was is None else "removed upstream" + continue + fields = [ + f"{label}: {a!r} -> {b!r}" if label != "value" else f"value: {digest(a)} -> {digest(b)}" + for label, a, b in zip(("section", "class", "value"), was, is_) + if a != b + ] + out[name] = "; ".join(fields) + return out def _declared_classes(registry: dict) -> dict[str, str]: """name -> the class the registry DECLARES for it (never inferred from the name).""" - return { - name: body.get("class") - for section in ("connection", "target", "secret") - for name, body in registry.get(section, {}).items() - } + return {n: row[1] for n, row in _rows(registry, _CONSUMED_TABLES).items()} @pytest.mark.parametrize("partner", _PARTNERS) def test_every_declared_name_exists_in_the_registry_with_the_class_we_treat_it_as(partner): """C-57: a rename or reclassification upstream must not be silent here.""" repo = require_sibling("views-appwrite") - declared = _declared_classes(_load_registry(repo)) + declared = _declared_classes(_registry_current(repo)) _, _, expected_class = _PARTNER_ENV[partner] missing = sorted(n for n in expected_class if n not in declared) @@ -460,24 +646,165 @@ def test_every_declared_name_exists_in_the_registry_with_the_class_we_treat_it_a @pytest.mark.parametrize("partner", _PARTNERS) -def test_the_pinned_contract_edition_still_matches_the_registry(partner): - """The check that catches everything the other one cannot — including additions. +def test_the_pinned_commit_declares_the_pinned_version(partner): + """The pair is a claim, and until now nothing checked it. + + ``appwrite_env.py`` declares a version AND a commit, and the old drift check's own + message said *"Do not bump one alone — the pair is the claim."* That sentence lived + only inside an error string: no test compared the two. Someone could bump either + alone and the whole suite stayed green, leaving a pin that names an edition nobody + published. + + Verified 2026-08-11: ``fcf32c9`` does declare ``1.4.4`` — true by luck, not by check. + + Both sides of the *comparison* are frozen, so upstream editing the registry cannot + change its verdict. It still needs the sibling checkout to read the pinned blob, so a + missing or degraded clone reddens it — the check that runs with no sibling at all is + ``test_the_docstring_states_the_same_edition_the_constants_declare``. + """ + repo = require_sibling("views-appwrite") + module = _PARTNER_ENV[partner][0] + pinned = _registry_at(repo, module.SEAM_CONTRACT_COMMIT) + assert pinned["meta"]["version"] == module.SEAM_CONTRACT_VERSION, ( + f"[{partner}] the pin is internally inconsistent: commit " + f"{module.SEAM_CONTRACT_COMMIT} declares registry v{pinned['meta']['version']}, " + f"but SEAM_CONTRACT_VERSION says v{module.SEAM_CONTRACT_VERSION}. One of the two " + "was bumped alone. The pair is the claim — fix whichever is wrong." + ) + + +def test_every_table_in_the_registry_is_classified_here(): + """A table nobody classified is an upstream change nobody looked at. + + This replaces the old edition-equality check, and it is worth being precise about + what that check was really doing. It compared version *strings*, so it fired on every + edition — five in four days, four of them recording console observations that carried + no obligation for anyone here (register C-86). A guard that cries wolf gets bumped + reflexively, which is how its one real firing goes unread. - Names and classes catch a rename. The edition catches **any** other change: a new - target this repo ought to adopt, a retired secret slot, a reworded rule. It fails - loudly and tells you what to do rather than what broke. + But it did have a real job underneath the noise, and this is that job. On 2026-08-10 + the registry grew a ``[contract.*]`` table carrying a live obligation for this + repository — the ADR-017 delivery-label mirror — and the edition check was the *only* + mechanism here that noticed, because ``_declared_classes`` parses three tables and + was silently ignoring four. - It is also the check that would have caught #211's stale crafd pin on the day it - was written, had it been looking at crafd. It was not. It is now. + **Directional on purpose.** Every table upstream must be classified; only the tables + we depend on must exist. An IGNORED table disappearing is not our business, which is + an IGNORED table disappearing is silent while a new, unclassified one is a red build + with something to do. Two such events in the registry's life so far, and this + repository needed to see both. + + *(An earlier draft illustrated the silent case with v1.5.1's removal of + ``[unmodelled]``. That was wrong — ``unmodelled`` was never in the partition, so it + would have been a red build demanding classification, not a silent pass.)* + """ + repo = require_sibling("views-appwrite") + registry = _registry_current(repo) + + unclassified = _unclassified_tables(registry) + assert not unclassified, ( + f"the registry has table(s) this repository has never classified: " + f"{unclassified}. Decide what each one is — CONSUMED (we read it), MIRRORED (its " + "values live in our source), or IGNORED (with a reason) — and add it to " + "_TABLE_ROLE. A new table is how an obligation arrives; `[contract.*]` arrived " + "exactly this way and only a noisy version check happened to catch it." + ) + vanished = _missing_dependencies(registry) + assert not vanished, ( + f"table(s) this repository depends on are gone from the registry: {vanished}. " + "Either they were retired upstream and this package must stop reading them, or " + "the file being read is not the registry." + ) + + +@pytest.mark.parametrize("partner", _PARTNERS) +def test_nothing_this_repo_reads_has_changed_since_the_pin(partner): + """The drift check, matching on the facts rather than on the edition label. + + ADR-014 §3: when a guard fires on something legitimate, ask whether the *matching* is + wrong before you narrow the scope. The old check matched on ``meta.version``, which + is a label for "anything at all changed". This matches on the rows this package + actually declares, compared between the edition we pinned and the edition upstream + has now. + + **Fires on:** a rename, a reclassification, a removal, a valueless row gaining a + value — and on **rotation**, which is the one no other check in this file can see. A + rotated bucket id keeps its name and its class, so names-and-classes passes while + every delivery goes somewhere else. The no-copy rule forbids writing the expected + values into this repository, so the pinned edition is the only lawful baseline for + that comparison. + + **Silent through:** prose edits, ``[meta]`` bumps, and rows belonging to anyone else. + Measured across the real v1.4.4 -> v1.5.2 window (three editions, one week): green. + + **The stopping rule, because this check has been widened once already.** It fires on + exactly one condition: *a row named in this partner's* ``expected_class`` *differs + between the pinned edition and the sibling's* ``main``. + + Any proposal to widen it must state two things: which delivery failure it prevents, + **and its false-alarm surface measured against the live registry**. If that surface + includes rows this package does not read, it is refused. (An earlier version of this + rule asked instead whether ``assert_env_declared`` already prevented the failure at + run time. That rule is wrong in both directions: an unadopted upstream coordinate is + not prevented by it, so arrival detection would pass — and a removed coordinate *is* + caught by it, so the removal detection this check does perform would fail.) + + An arrival half failed the surface test and was deleted (#245): it compared every row + of every depended-on table, and measured, each partner reads 13 of 25 while 6 belong + to no repository here. It defended itself by claiming ``[contract.*]`` + "arrived exactly this way and nothing else here would have seen it" — false at table + granularity, where ``test_every_table_in_the_registry_is_classified_here`` catches it. + + **What that deletion costs, stated rather than implied.** A coordinate views-appwrite + issues *for this package* is now silent until a human reads the registry — no test, no + run-time assert, nothing. So is a second ``[contract.*]`` row for a future partner. + That is the accepted price of not being reddened by every unrelated row, and it is + carried in register C-90 rather than here. """ repo = require_sibling("views-appwrite") + module, _, expected_class = _PARTNER_ENV[partner] + pinned = _registry_at(repo, module.SEAM_CONTRACT_COMMIT) + current = _registry_current(repo) + + names = set(expected_class) + then, now = _projection(pinned, names), _projection(current, names) + + changed = _describe_changes(then, now, names) + assert not changed, ( + f"[{partner}] the registry changed something this package reads, since the " + f"edition it was verified against (v{module.SEAM_CONTRACT_VERSION}, commit " + f"{module.SEAM_CONTRACT_COMMIT}): {changed}. Re-verify this module's declaration " + "against the current edition, then move SEAM_CONTRACT_VERSION and " + "SEAM_CONTRACT_COMMIT together. If a value rotated, the delivery is pointing " + "somewhere else until the launcher's environment is updated too." + ) + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_the_docstring_states_the_same_edition_the_constants_declare(partner): + """The module says the edition twice — in prose and in a constant. Only one is checked. + + Both `appwrite_env.py` docstrings name the registry edition in a sentence + (*"That pin is registry v1.4.1"*) beside the constant that declares it. The drift + detector reads the constant, so on 2026-08-05 the pins moved to v1.4.4 with both + guards green and both docstrings still saying v1.4.1 — a reader following the prose + would have checked their coordinates against a superseded edition. + + This is the same defect the register carries as C-80 and C-82: a claim in prose next + to a fact in code, with a guard on the fact only. Cheap to close here because the + prose states the value in a fixed form, so the two can simply be compared. + """ module = _PARTNER_ENV[partner][0] - actual = _load_registry(repo)["meta"]["version"] - assert actual == module.SEAM_CONTRACT_VERSION, ( - f"the Appwrite Seam Contract's registry moved to v{actual}; " - f"{partner}/appwrite_env.py declares v{module.SEAM_CONTRACT_VERSION}. Re-verify " - f"that module's declaration against v{actual}, then bump SEAM_CONTRACT_VERSION " - "and SEAM_CONTRACT_COMMIT together. Do not bump one alone — the pair is the claim." + stated = re.findall(r"registry \*\*v([\d.]+)\*\*", module.__doc__ or "") + assert stated, ( + f"{partner}/appwrite_env.py's docstring no longer states the registry edition in " + "the form this guard reads. If the sentence was reworded, reword the pattern too " + "— do not delete the check, or the prose goes unguarded again." + ) + assert set(stated) == {module.SEAM_CONTRACT_VERSION}, ( + f"[{partner}] the docstring says registry v{'/v'.join(sorted(set(stated)))} but " + f"SEAM_CONTRACT_VERSION declares v{module.SEAM_CONTRACT_VERSION}. The constant is " + "what the drift detector checks, so the prose is the half that rots silently." ) @@ -505,6 +832,227 @@ def test_the_pinned_commit_is_reachable_from_the_contract_repos_main(partner): ) +def test_the_pinned_reader_refuses_every_way_a_baseline_can_be_wrong(tmp_path): + """Each refusal branch, against a scratch repository built to trigger it. + + The first version of this test used a bogus sha against *this* repository — where the + registry path does not exist at any ref, so a nonsense sha and ``HEAD`` failed + identically. It proved the helper rejects *something*, not that it rejects an + unreadable **pin**, and deleting the empty-file and anchor branches left it green. + + Every branch here is genuinely reachable, and each is the shape of a real accident: + a pin blanked by a bad edit, a pin pointing at a branch, a pin naming an annotated + tag, a file emptied upstream, a file that parses but is not the registry. + """ + import subprocess as sp + + def git(*args): + return sp.run(["git", "-C", str(tmp_path), *args], capture_output=True, text=True, check=True) + + git("init", "-q") + git("config", "user.email", "t@t") + git("config", "user.name", "t") + target = tmp_path / _REGISTRY_RELPATH + target.parent.mkdir(parents=True) + + def commit(text: str, message: str) -> str: + target.write_text(text) + git("add", "-A") + git("commit", "-q", "-m", message) + return git("rev-parse", "--short", "HEAD").stdout.strip() + + good = commit('[meta]\nversion = "9.9.9"\n\n[connection.X]\nclass = "connection"\n', "good") + empty = commit("", "empty") + anchorless = commit('[target.X]\nclass = "target"\n', "no meta, no connection") + git("tag", "-a", "v1", "-m", "annotated") + + assert _registry_at(tmp_path, good)["meta"]["version"] == "9.9.9", ( + "the reader must accept a well-formed registry, or the refusals below prove nothing" + ) + + with pytest.raises(_RegistryReadError, match="does not resolve to a commit"): + _registry_at(tmp_path, "") # a blanked pin reads the INDEX + with pytest.raises(_RegistryReadError, match="annotated TAG|does not start with it"): + _registry_at(tmp_path, "v1") # a tag object: git peels it, the URL 404s + with pytest.raises(_RegistryReadError, match="is EMPTY"): + _registry_at(tmp_path, empty) # exits 0, stdout empty + with pytest.raises(_RegistryReadError, match="no meta.version or no"): + _registry_at(tmp_path, anchorless) # parses, but is not the registry + with pytest.raises(_RegistryReadError, match="does not resolve to a commit"): + _registry_at(tmp_path, "0" * 40) # a pin that names nothing + + +def test_the_role_vocabulary_is_closed(): + """A typo in a role string used to silently narrow a security scan. + + ``_TABLE_ROLE``'s values drive which sections the no-copy check reads. Mistyping + ``"CONSUMED"`` dropped `target` from that scan — twelve coordinate values to two — + with nothing objecting, while also narrowing what counts as a dependency. Roles are + now a closed set, asserted. + """ + unknown = sorted({r for r in _TABLE_ROLE.values() if r not in _ROLES}) + assert not unknown, ( + f"unknown role(s) in _TABLE_ROLE: {unknown}. Roles drive which sections the " + f"no-copy scan reads; a typo here silently narrows it. Use one of {_ROLES}." + ) + assert _CONSUMED_TABLES, "no table is classified CONSUMED — every row check reads nothing" + + +def test_the_table_partition_would_catch_a_new_table_and_a_vanished_one(): + """Calls the production predicates, so neutering them fails here. + + The first version of this test re-typed the set arithmetic inline. Mutation-proven + afterwards: replacing the real computations with ``[]`` left the guard accepting any + upstream registry forever, and this proof still passed. It detected nothing, which is + ADR-014 §2 exactly — and it was the ONLY thing claiming the table check bites. + + The asymmetry is the design and is asserted in both directions: an unclassified table + fires, a vanished IGNORED table does not. + """ + base = {name: {} for name in _TABLE_ROLE} + + assert _unclassified_tables(base | {"brand_new_table": {}}) == ["brand_new_table"], ( + "a table nobody classified went unnoticed — that is how `[contract.*]` arrived" + ) + assert not _unclassified_tables(base), "the real registry's tables must all classify" + + assert _missing_dependencies({k: v for k, v in base.items() if k != "target"}) == ["target"], ( + "a table this package reads rows out of vanished and the check did not object" + ) + assert not _missing_dependencies({k: v for k, v in base.items() if k != "excluded"}), ( + "an IGNORED table disappearing must NOT fire (ADR-014 §3: prefer the false " + "negative to the false alarm)" + ) + + +@pytest.mark.parametrize("partner", _PARTNERS) +def test_the_drift_check_would_catch_a_rotation_that_names_and_classes_cannot(partner): + """Rotation is why the projection carries ``value`` — proven through the real code. + + The first version of this test asserted ``then != now`` on dicts it built itself. + Mutation-proven afterwards: making the production comparison ignore ``value`` — which + is *literally* "names and classes cannot see a rotation", the thing this test exists + to disprove — left both the proof and the check green, while a rotated bucket id + shipped every delivery to the wrong place. So it now calls ``_describe_changes``, the + same function the check calls. + + Runs without a sibling, so it holds on every machine rather than only where + views-appwrite happens to be checked out. + """ + _, own_store, expected_class = _PARTNER_ENV[partner] + canary = own_store[0] + names = set(expected_class) + + def edition(value: str) -> dict: + return { + "meta": {"version": "0.0.0-fixture"}, + "connection": {"APPWRITE_ENDPOINT": {"class": "connection", "value": "e"}}, + "target": {canary: {"class": "target", "value": value}}, + } + + was, rotated = "at-the-pin", "after-rotation" + then = _projection(edition(was), names) + now = _projection(edition(rotated), names) + + assert then[canary][:2] == now[canary][:2], ( + "this fixture must differ ONLY in the value, or it is not proving what it claims" + ) + changed = _describe_changes(then, now, names) + assert canary in changed, ( + f"[{partner}] a rotated value went unnoticed. Name and class are identical on " + "both sides, so nothing else in this file can see it." + ) + assert "value:" in changed[canary], ( + f"[{partner}] the report must name the FIELD that moved, or a maintainer reading " + "the failure cannot tell a rotation from a reclassification." + ) + # Both sides, from the fixture's own variables — the rotated one was never checked. + for side in (was, rotated): + assert side not in changed[canary], ( + f"[{partner}] the report printed the {side!r} value. It must name the field " + "and never the value: this repository is public, its CI logs are " + "world-readable, and they cannot be redacted afterwards." + ) + + +@pytest.mark.parametrize("partner", _PARTNERS) +def test_the_drift_check_is_silent_on_a_row_this_partner_does_not_read(partner, monkeypatch): + """The stopping rule above, as a check rather than a paragraph. + + Paired with ``test_the_drift_check_fires_when_a_row_this_partner_reads_rotates``, + which is the positive direction through the same harness. Neither is optional: alone, + this one is a check that can only pass, and alone the twin says nothing about what the + check ignores. + + An earlier version of this docstring vouched for the two proofs either side — "rotation + fires, a rename fires" — and neither is about this check. One calls the helper + underneath it; the other belongs to the class check. That is this file's own recurring + defect (C-80, C-82), so the vouching is gone and the twin is real. + + **It drives the real check**, through the real registry readers, rather than calling + the projection helpers underneath it. A first version called ``_describe_changes`` + directly and was worthless: re-adding the deleted arrival half to the check left the + whole suite green, because that half never lived in the helper this was asking. A + guard has to be pointed at the thing it claims to guard. + """ + expected_class = _PARTNER_ENV[partner][2] + targets = sorted(n for n, cls in expected_class.items() if cls == "target") + assert targets, ( + f"[{partner}] declares no coordinate of class 'target', so this fixture has " + "nothing to file under [target] and would prove nothing. (An earlier version " + "asserted that the chosen name's class was 'target' — true by construction, " + "since the name was selected by that condition. A proof that cannot fail is the " + "defect this entry's own C-90 records.)" + ) + mine = targets[0] + + pinned = { + "meta": {"version": "0.0.0-fixture"}, + "connection": {"APPWRITE_ENDPOINT": {"class": "connection", "value": "e"}}, + "target": {mine: {"class": "target", "value": "unchanged"}}, + } + # The same registry, plus one coordinate belonging to another repository. + current = {**pinned, "secret": {"SOMEBODY_ELSES_API_KEY": {"class": "secret"}}} + + here = sys.modules[__name__] + monkeypatch.setattr(here, "require_sibling", lambda name: Path("/nonexistent")) + monkeypatch.setattr(here, "_registry_at", lambda repo, ref: pinned) + monkeypatch.setattr(here, "_registry_current", lambda repo: current) + + test_nothing_this_repo_reads_has_changed_since_the_pin(partner) + + +@pytest.mark.parametrize("partner", _PARTNERS) +def test_the_drift_check_fires_when_a_row_this_partner_reads_rotates(partner, monkeypatch): + """The positive twin of the silence guard, through the same real check. + + Without this, the check's entire ``assert not changed`` could be deleted and the suite + would stay green — measured. Its two apparent proofs are not about it: the rotation + proof above calls ``_describe_changes``, the helper underneath, and the rename proof + calls ``_declared_classes``, which belongs to a different check altogether. So the + silence guard was the only test driving this one, and a guard that can only pass is + the same defect as a guard pointed at the wrong subject. + """ + targets = sorted(n for n, cls in _PARTNER_ENV[partner][2].items() if cls == "target") + assert targets, f"[{partner}] declares no 'target' coordinate to rotate" + mine = targets[0] + + pinned = { + "meta": {"version": "0.0.0-fixture"}, + "connection": {"APPWRITE_ENDPOINT": {"class": "connection", "value": "e"}}, + "target": {mine: {"class": "target", "value": "at-the-pin"}}, + } + rotated = {**pinned, "target": {mine: {"class": "target", "value": "after-rotation"}}} + + here = sys.modules[__name__] + monkeypatch.setattr(here, "require_sibling", lambda name: Path("/nonexistent")) + monkeypatch.setattr(here, "_registry_at", lambda repo, ref: pinned) + monkeypatch.setattr(here, "_registry_current", lambda repo: rotated) + + with pytest.raises(AssertionError, match=mine): + test_nothing_this_repo_reads_has_changed_since_the_pin(partner) + + @pytest.mark.parametrize("partner", _PARTNERS) def test_the_drift_check_would_catch_a_rename(partner): """A gated test that cannot fail is decoration — so prove this one bites in CI. @@ -563,43 +1111,55 @@ def _docstring_nodes(tree: ast.AST) -> set[int]: return out + def test_no_coordinate_value_is_copied_into_this_repo(): """The registry's own rule: *"never bake a value into code, an example, or a dataclass default."* Consumers READ and VALIDATE; the launcher supplies values. - **Two wrong narrowings before this one, both instructive.** - - A substring text scan over every ``.py`` flagged three "leaks": ``file_metadata`` - (a **function name** in ``contract/store_metadata.py``), ``production_forecasts`` - and ``unfao_bucket`` (only in refusal labels and docstrings naming which store a - function serves). None was a copy, and a guard that fails on - ``def file_metadata(record)`` gets deleted — after which the real rule is unguarded. - - Narrowing to *assignments and default arguments* then went too far in the other - direction: it caught neither a dict value nor a keyword argument, and the keyword - argument is the shape this repo would actually produce — - ``AppwriteConfig(bucket_id=...)`` is how every store is configured, and swapping one - ``os.getenv`` for a literal there is the violation. + **Names coordinates, never values** — including in this docstring. pytest prints the + failing function's source, so a value written here publishes on exactly the event the + check exists to catch (register C-89). - The right axis was **exact equality on string constants**, not statement shape. It - catches dict values and kwargs, while all three original false positives fall out on - their own: a function name is not a ``Constant``; ``"unfao_bucket datastore"`` is not - equal to ``"unfao_bucket"``; docstrings are excluded outright. + The axis is **exact equality on string constants**, not statement shape: it catches + dict values and keyword arguments — ``AppwriteConfig(bucket_id=...)`` is how every + store is configured — while a function name is not a ``Constant`` and docstrings are + excluded outright. Two narrower drafts and their false positives are in register + C-57; three values that cannot be told from ordinary code at all are in C-97. """ - repo = sibling_repo("views-appwrite") - if repo is None: - pytest.skip("views-appwrite checkout not found — set VIEWS_APPWRITE") - registry = _load_registry(repo) - values = { - body["value"] - for section in ("connection", "target") - for body in registry.get(section, {}).values() - if isinstance(body.get("value"), str) and len(body["value"]) > 6 - } - + # `require_sibling`, like every other reader in this file. The hand-rolled skip that + # stood here said only "checkout not found — set VIEWS_APPWRITE", which is the + # degraded message `require_sibling` exists to replace: it names the variable but not + # the conventional path, so a contributor can watch it skip and not run it. Being the + # last call site off the shared path is how a rule ends up with two behaviours. + repo = require_sibling("views-appwrite") + registry = _registry_current(repo) + # Scoped by the declared partition, NOT by an inline tuple — so a new table cannot + # be swept in by a one-word edit, and `contract` cannot be swept in at all. + # + # `[contract.*]` values are MIRRORED: ADR-017 §5 requires them to appear in this + # package's source, because we write them onto every upload. Banning them here would + # forbid the thing the contract obliges. That is not an exception to "never copy a + # coordinate" — it is a different class, declared upstream: the registry's own header + # says no reader scans `[contract.*]`, so no value there ever reaches a process + # environment. The no-copy rule protects values the launcher supplies; a mirror is + # the inverse by construction. + scanned_sections = tuple( + name for name, role in _TABLE_ROLE.items() if role == "CONSUMED" + ) + # value -> every coordinate declaring it. A list, because two coordinates may share + # a value and measured against the live registry two pairs do. + declared_by_value: dict[str, list[str]] = {} + for section in scanned_sections: + for name, body in registry.get(section, {}).items(): + # No length floor: it excluded two five-character coordinates and, + # measured, removing it keeps the suite green — so it was buying nothing + # while narrowing a security check. + if isinstance(body.get("value"), str) and body["value"].strip(): + declared_by_value.setdefault(body["value"], []).append(name) + values = set(declared_by_value) copied = [] for source in sorted(_PKG.rglob("*.py")): - tree = ast.parse(source.read_text()) + tree = _parsed(source) docstrings = _docstring_nodes(tree) for node in ast.walk(tree): if ( @@ -608,7 +1168,11 @@ def test_no_coordinate_value_is_copied_into_this_repo(): and node.value in values and id(node) not in docstrings ): - copied.append(f"{source.relative_to(_PKG)}:{node.lineno} = {node.value!r}") + names = ", ".join(sorted(declared_by_value[node.value])) + copied.append( + f"{source.relative_to(_PKG)}:{node.lineno} carries the value " + f"declared for {names}" + ) # Markdown too — the AST half cannot see a fenced ``bash`` block, and that is exactly # where four production-forecasts values sat: in README.md's Configuration section, # two lines below the sentence promising they are never copied, in a PUBLIC @@ -617,25 +1181,27 @@ def test_no_coordinate_value_is_copied_into_this_repo(): # # **What counts as a copy, and what does not.** A first draft flagged any line # containing a registry value and immediately fired on a dozen documents that merely - # *name* a store in prose — "six stranded documents in unfao_bucket". That is not a + # *name* a store in prose — a sentence counting stranded documents in the UNFAO + # bucket, written with the bucket's declared value. That is not a # copy; it is a sentence. C-57 recorded the identical false-positive class over `.py` # and the identical lesson: when a guard cries wolf, the matching is wrong before the # scope is (ADR-014 §3). # - # The copy is a value **assigned to its own coordinate name** — `APPWRITE_X=value` — - # which is a reader's instruction to configure with that literal. That is precise - # enough to have caught README.md and to ignore every legitimate mention. - assignment = re.compile( - r"^\s*(?:export\s+)?(" + "|".join(sorted(_EXPECTED_NAMES)) + r")\s*=\s*(.+?)\s*$" - ) - # This repository's OWN tracked markdown — `git ls-files`, not `rglob`. CI checks - # sibling repositories out into the workspace, and their documents are not this - # repo's to police; an rglob would scan them and fail on someone else's prose. + # A copy is a declared NAME assigned a declared VALUE. Both halves are known before + # the scan starts, so it matches the pair rather than parsing the line and comparing + # what it captured. Three rounds of blindness came from that parse, all of them the + # same shape — the pattern described one dialect of markdown (register C-57). # - # **This half cannot run in CI**, and that is worth stating rather than discovering. - # The registry values come from the views-appwrite checkout, which is private and - # deliberately not checked out — so the whole test skips there. It guards a - # maintainer's commit, not the merge. Closing that is C-81's token decision. + # The two halves need not correspond: `APPWRITE_X=` is + # the ordinary copy-paste slip, and is a copy. + # + # It stays a syntax match. A value merely *named* in a sentence is not a copy, and a + # draft that fired on those was deleted for crying wolf (ADR-014 §3, C-57). + names_alt = "|".join(sorted(_EXPECTED_NAMES)) + values_alt = "|".join(re.escape(v) for v in sorted(values, key=len, reverse=True)) + pair = re.compile( + rf"""(?@', got {version!r} — the delivery " + "carries this verbatim into its provenance record (C-15)." + ) diff --git a/tests/test_gaul_lookup_fidelity.py b/tests/test_gaul_lookup_fidelity.py index 870672d..ee023c4 100644 --- a/tests/test_gaul_lookup_fidelity.py +++ b/tests/test_gaul_lookup_fidelity.py @@ -37,6 +37,7 @@ from pathlib import Path import numpy as np +import pyarrow as pa import pyarrow.parquet as pq import pytest @@ -217,10 +218,16 @@ def test_lookup_declares_its_provenance(lookup_meta): def test_lookup_version_stamp_resolves(lookup): - """The stamp the delivery provenance carries (C-15) must not be 'unknown'.""" - from views_postprocessing.contract.enrichment import GaulLookupEnricher + """The stamp the delivery provenance carries (C-15) must not be 'unknown'. - version = GaulLookupEnricher(_LOOKUP).lookup_version + Read through ``gaul_lookup.version`` — the declared reader the delivery itself uses. + It was read through ``GaulLookupEnricher`` until that object was retired (#90, C-75); + the enricher was never on the delivery path, so this now checks the same fact through + the code that ships. + """ + from views_postprocessing.contract import gaul_lookup + + version = gaul_lookup.version(_LOOKUP) assert version != "unknown", ( "lookup_version resolved to 'unknown' — the delivery would ship untraceable " "provenance (C-60). The lookup's embedded source_provenance is absent or reshaped." @@ -302,22 +309,24 @@ def test_coordinate_formula_matches_every_priogrid_cell(): # ── the builder's own invariants must not be vacuous ──────────────────────── -def _synthetic_source(n: int = 6): - """A minimal well-formed source frame in the producer's column vocabulary.""" - import pandas as pd +def _synthetic_source(n: int = 6) -> pa.Table: + """A minimal well-formed source table in the producer's column vocabulary. - gids = list(range(1, n + 1)) - return pd.DataFrame( + Arrow since #90 — ``_load_source`` returns a table with a ``gid`` COLUMN rather than + a frame with a ``gid`` index, because arrow has no index. The builder's contract with + this seam is the column vocabulary, which is unchanged. + """ + return pa.table( { - "gaul0_code": [10] * n, - "gaul0_name": ["Country"] * n, - "gaul1_code": [20] * n, - "gaul1_name": ["Admin1"] * n, - "gaul2_code": [30] * n, - "gaul2_name": ["Admin2"] * n, - "iso3_code": ["ABC"] * n, - }, - index=pd.Index(gids, name="gid"), + "gid": pa.array(range(1, n + 1), type=pa.int64()), + "gaul0_code": pa.array([10] * n, type=pa.int64()), + "gaul0_name": pa.array(["Country"] * n), + "gaul1_code": pa.array([20] * n, type=pa.int64()), + "gaul1_name": pa.array(["Admin1"] * n), + "gaul2_code": pa.array([30] * n, type=pa.int64()), + "gaul2_name": pa.array(["Admin2"] * n), + "iso3_code": pa.array(["ABC"] * n), + } ) @@ -349,16 +358,16 @@ def _build_with(monkeypatch, source, tmp_path, provenance=None): def test_builder_rejects_a_duplicate_gid(monkeypatch, tmp_path): """C-59: prove the uniqueness guard is not vacuous. - Note the guard is only *reachable* with ``region="all"``. Under a region filter - pandas' ``Index.intersection`` silently de-duplicates first — an accidental + Note the guard is only *reachable* with ``region="all"``. Under a region filter the + old pandas build de-duplicated silently via ``Index.intersection`` — an accidental protection, not a declared one, which is why the explicit raise still earns its - place. See the C-59 narrative for the corrected exposure analysis. + place. Since #90 the filter is ``pc.is_in``, which does **not** de-duplicate, so the + raise is the only thing standing there. See the C-59 narrative. """ - import pandas as pd import scripts.build_gaul_lookup as builder source = _synthetic_source() - duplicated = pd.concat([source, source.iloc[[2]]]) + duplicated = pa.concat_tables([source, source.slice(2, 1)]) with pytest.raises(builder.LookupBuildError, match="duplicate gid"): _build_with(monkeypatch, duplicated, tmp_path) @@ -379,17 +388,48 @@ def test_a_sentinel_code_is_dropped_rather_than_shipped(monkeypatch, tmp_path): pandas internals, and a test that fragile is worse than the invariant it guards. """ source = _synthetic_source() - source.loc[3, "gaul1_code"] = -1 + codes = source.column("gaul1_code").to_pylist() + codes[2] = -1 + source = source.set_column( + source.column_names.index("gaul1_code"), "gaul1_code", + pa.array(codes, type=pa.int64()), + ) result = _build_with(monkeypatch, source, tmp_path) - assert 3 not in result.index, "the -1 cell must be dropped from the lookup" - assert len(result) == 5 + assert 3 not in result.column("priogrid_gid").to_pylist(), ( + "the -1 cell must be dropped from the lookup" + ) + assert result.num_rows == 5 + + +def test_builder_refuses_a_build_with_zero_cells(monkeypatch, tmp_path): + """Register C-76 — an empty lookup is writable and looks like a result. + + A ``--region`` that filters every cell out, or a join with no overlap across the + seven sources, produced a zero-row parquet and printed ``cells=0`` as though that + were an outcome. It failed later and confusingly: ``build_historical_table`` raises + about *missing geography*, not about an empty lookup, and by then the artifact is + committed. + + The first draft of the guard did not survive its own test. On an empty table + ``pa.array([True] * 0)`` infers NULL type, so ``pc.and_`` raised + ``ArrowNotImplementedError`` in the completeness filter — before the zero-row check + could say anything. The mask is now typed, and this asserts the refusal is the one a + human can act on. + """ + import scripts.build_gaul_lookup as builder + + empty = _synthetic_source(0) + with pytest.raises(builder.LookupBuildError, match="ZERO cells"): + _build_with(monkeypatch, empty, tmp_path) def test_builder_accepts_a_clean_source(monkeypatch, tmp_path): """The guards must not reject well-formed input.""" result = _build_with(monkeypatch, _synthetic_source(), tmp_path) - assert len(result) == 6 - assert list(result.columns) == METADATA_COLS + assert result.num_rows == 6 + # The nine declared columns in normative order, with the key last — the artifact's + # shipped shape, which #90 had to preserve exactly while changing what builds it. + assert result.column_names == METADATA_COLS + ["priogrid_gid"] @_needs_datafactory diff --git a/tests/test_input_integrity_design_contract.py b/tests/test_input_integrity_design_contract.py index b86416e..125fc8a 100644 --- a/tests/test_input_integrity_design_contract.py +++ b/tests/test_input_integrity_design_contract.py @@ -79,9 +79,16 @@ def test_manager_does_not_inherit_forecasting_model_manager(): # housekeeping -------------------------------------------------------------- -def test_enrichment_does_not_call_a_dataframe_a_prediction_frame(): - src = (_PKG / "contract" / "enrichment.py").read_text().lower() - assert "prediction frame" not in src +# `test_enrichment_does_not_call_a_dataframe_a_prediction_frame` was removed with +# `contract/enrichment.py` in #90 (register C-75). It asserted that one module did not +# call a DataFrame a "prediction frame" — vocabulary hygiene in a module that no longer +# exists. +# +# Nothing replaces it, and that is the honest position rather than an oversight: the +# confusion it guarded against was possible only while a pandas object sat next to a +# frame-shaped API. `test_delivery_invariants_are_pandas_free` above, and the +# now-empty type-only-pandas assertion in `test_doc_accuracy.py`, together mean there +# is no DataFrame left in the package to mislabel. def test_no_lingering_mapping_directory(): diff --git a/tests/test_product.py b/tests/test_product.py index 8cdab80..e4b461d 100644 --- a/tests/test_product.py +++ b/tests/test_product.py @@ -8,9 +8,11 @@ The consumer-name pin is the one that matters most and the one a partner-scoped test could never catch: a document written under a name the consumer does not filter for is -delivered, stored, billed, and invisible. That is not hypothetical here — six -`orange_ensemble` forecast documents sat in `unfao_bucket` for months while FAO's -forecast serving read empty (ADR-013 §11.4 post-adoption record, register C-01). +delivered, stored, billed, and invisible. That is not hypothetical here — six forecast +documents sat in the UNFAO bucket for months, under a name nothing was looking for, while +FAO's forecast serving read empty (ADR-013 §11.4 post-adoption record, register C-01). +*(Named, not quoted: the bucket's declared value is a coordinate, and this docstring is +printed by pytest whenever anything in this file fails — register C-97.)* Parametrised over `tests/conftest.PARTNER_PACKAGES`, so a third partner is caught by `test_clone_readiness.py::test_the_declared_partner_list_is_the_real_one` rather than @@ -25,7 +27,8 @@ import pytest -from tests.conftest import CONSUMER_REPO, PARTNER_PACKAGES, SIBLING_ENV, require_sibling +from tests.conftest import CONSUMER_REPO, PARTNER_PACKAGES, SIBLINGS, require_sibling +from tests.seam_registry import registry_current #: partner -> the document name its consumer filters on, DECLARED here rather than #: read back from the module under test. @@ -34,44 +37,32 @@ #: passes for every possible value, which is the shape of the replica defect epic #181 #: spent a story retiring. So the expected value is written out. #: -#: **But a literal transcribed from another repository is a guarantee, and ADR-014 §1 -#: says a guarantee needs a check.** On its own this pin catches drift authored *here* — -#: which was never the danger. If views-crafdapi changed its filter tomorrow, this file -#: would stay green, the delivery would upload, and the document would be invisible: -#: ADR-013 §4.1a, the defect that stranded six `orange_ensemble` documents in -#: `unfao_bucket` while FAO's forecast serving read empty for months. +#: **A literal transcribed from another repository is a guarantee, and ADR-014 §1 says a +#: guarantee needs a check.** On its own this pin catches drift authored *here*, which was +#: never the danger: if a consumer changed its filter tomorrow this file would stay green, +#: the delivery would upload, and the document would be invisible (ADR-013 §4.1a). #: -#: ``test_the_declared_consumer_name_still_matches_the_consumer`` closes that half -#: against the sibling checkout, following the pattern -#: ``tests/test_env_declaration.py`` already uses for the coordinate registry: declared -#: locally, verified across the seam when the other repo is on disk. +#: ``test_the_declared_consumer_name_matches_the_registry`` closes that half against the +#: **public coordinate registry** rather than the consumer's source (ADR-017 §5). It runs +#: for every partner and needs no credential, because the registry is public even when the +#: consumer is not — which is why it reaches the FAO partner and its predecessor could not. +#: +#: What it does **not** close is whether the consumer's query uses the name it declares. +#: This repository used to read their source for that; the check broke twice in 24 hours +#: because both consumers refactored a literal into a named constant, and #248 deleted it +#: rather than repairing it. Register **C-92** carries the gap; views-faoapi#390 and +#: views-crafdapi#55 are the asks that would close it where the fact lives. _CONSUMER_DOCUMENT_NAME = { "unfao": "un_fao", "crafd": "un_crafd", } -#: Where the consumer's name lives, and the mechanism that consumes it. Both are -#: pinned: a consumer that kept the string but started filtering on a different field -#: would strand a delivery just as thoroughly as one that renamed it. -_CONSUMER_PATH_MANAGER = re.compile(r'APIPathManager\(\s*"([a-z0-9_]+)"') -_CONSUMER_FILTER = 'filters["name"] = self.model_path.model_name' - +_PKG = Path(__file__).resolve().parent.parent / "views_postprocessing" def _product(partner: str): return importlib.import_module(f"views_postprocessing.{partner}.product") -def _consumer_package(repo: Path) -> Path: - """``src/views_api`` inside a consumer checkout.""" - candidates = sorted((repo / "src").glob("views_*api")) - assert len(candidates) == 1, ( - f"expected exactly one package under {repo}/src, found {candidates}. The " - "consumer's layout changed; this test's assumption about where to look is " - "part of what it asserts." - ) - return candidates[0] - - def test_every_partner_has_its_consumer_name_pinned(): """Assert this file's declared scope is the real one (ADR-014 §2).""" assert set(_CONSUMER_DOCUMENT_NAME) == set(PARTNER_PACKAGES), ( @@ -108,62 +99,160 @@ def test_upload_interlock_defaults_off(partner): assert _product(partner).UPLOAD_ENABLED is False -# ── across the seam: the pin above, checked against the repo that owns the fact ── +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_both_delivery_legs_name_the_document_from_the_declaration(partner): + """Register C-77 — the forecast and historical legs must agree, by construction. + + The two legs upload separately and the consumer selects them separately: forecasts + by the newest manifest, historical actuals by ``category="historical"``. Both are + filtered on the document ``name``, so if the legs disagree the delivery half-arrives + — and the failure mode is an **empty endpoint, not an error**, which is ADR-013 + §4.1a's exact shape. + + **They used to disagree.** The historical leg passed + ``name=self._model_path.model_name`` — the views-models *directory* name — while the + forecast leg passed the declared ``CONSUMER_DOCUMENT_NAME``. For FAO the two happen + to match, so nothing was wrong; nothing *asserted* they matched either, and the + agreement lived in a different repository's filesystem layout. The trap was about to + be sprung for real: views-models#333 creates CRAF'd's launcher directory, and + whoever named it would have decided, without knowing it, whether CRAF'd's historical + artifact was retrievable. + + A source scan rather than a call: the managers need Appwrite env and a views-models + path manager to instantiate, which is this repo's standing pattern for manager-side + facts. + """ + source = (_PKG / partner / "managers" / f"{partner}.py").read_text() + + assert "name=self._model_path.model_name" not in source, ( + f"[{partner}] an upload names its document from the model path again. That is " + "the views-models DIRECTORY name — a fact in another repository, not a " + "declaration here. Use product.CONSUMER_DOCUMENT_NAME (register C-77)." + ) + + # Counted separately, and the lookbehind matters: `consumer_name=product...` + # contains `name=product...` as a substring, so a naive count reports three legs + # where there are two. (It did, on the first run of this guard.) + legs = { + "forecast (via the sink)": len( + re.findall(r"\bconsumer_name=product\.CONSUMER_DOCUMENT_NAME", source) + ), + "historical (direct upload)": len( + re.findall(r"(? the registry row that is the AUTHORITY for its delivery label. +#: +#: Declared, never derived from the partner name: `unfao` -> `UNFAO_...` is mechanical, +#: `crafd` -> `UNCRAFD_...` is not, and guessing it would be the inference ADR-003 forbids. +_CONTRACT_ROW = { + "unfao": "UNFAO_CONSUMER_DOCUMENT_NAME", + "crafd": "UNCRAFD_CONSUMER_DOCUMENT_NAME", +} - The consumer decides what it filters on. This repository declares what it writes. - Agreement between the two is the whole of §4.1a, and it is not a fact this - repository owns — so it is checked against the consumer's checkout, exactly as the - coordinate registry is checked against views-appwrite's. - Gated: skips without the sibling, naming the variable to set. The always-on pin - above still runs everywhere, so CI is never left with nothing. +def test_every_partner_has_a_declared_contract_row(): + """Assert this file's declared scope is the real one (ADR-014 §2). + + A partner missing from ``_CONTRACT_ROW`` is a partner whose delivery label is checked + against nothing, and the suite stays green — which is exactly how ``crafd`` arrived + the first time. """ - repo = require_sibling(CONSUMER_REPO[partner]) - pkg = _consumer_package(repo) - - api = (pkg / "managers" / "api.py").read_text() - found = _CONSUMER_PATH_MANAGER.findall(api) - assert len(found) == 1, ( - f"expected exactly one APIPathManager(...) construction in " - f"{CONSUMER_REPO[partner]}'s managers/api.py, found {found}. More than one " - "means the consumer serves several document names and this check no longer " - "knows which one is ours." + assert set(_CONTRACT_ROW) == set(PARTNER_PACKAGES), ( + f"partners with no declared contract row: " + f"{sorted(set(PARTNER_PACKAGES) - set(_CONTRACT_ROW))}; declared but no longer a " + f"partner: {sorted(set(_CONTRACT_ROW) - set(PARTNER_PACKAGES))}." ) - assert found[0] == _CONSUMER_DOCUMENT_NAME[partner], ( - f"{CONSUMER_REPO[partner]} filters on {found[0]!r}; this repository declares " - f"{_CONSUMER_DOCUMENT_NAME[partner]!r} and writes it as " - f"{partner}/product.py's CONSUMER_DOCUMENT_NAME. A delivery under a name the " - "consumer does not ask for is uploaded, stored, and invisible (ADR-013 §4.1a)." + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_the_declared_consumer_name_matches_the_registry(partner): + """ADR-017 §5, the producer half: our mirror against the public declaration. + + **What this replaces, and why.** Until 2026-08-11 this check read the *consumer's + source* — a regex over `managers/api.py` looking for the argument to an + `APIPathManager(...)` construction. That worked on a laptop and never in CI for the + private partner, and it broke on 2026-08-11 when views-faoapi tidied that file into a + named constant. Their code got better and our check went looking for something that + had moved. ADR-017 §7: we were never entitled to depend on another repository's file + layout. + + Now both sides read one public declaration. The registry lives in views-appwrite, + which is public, so this needs no credential even for the private partner — that is + the whole of ADR-017 §5 in one assertion. + + *"Neither reads the other" is now the present state, not just the end state* — #248 + deleted the last source read. What that costs is register C-92: nothing here verifies + that a consumer's query uses the name it declares. + + The value it guards is the one whose failure is silent: a delivery filed under a name + the consumer does not ask for is uploaded, stored, billed, and invisible — no error + anywhere (ADR-013 §4.1a). + """ + repo = require_sibling("views-appwrite") + registry = registry_current(repo) + + contract = registry.get("contract") or {} + row = _CONTRACT_ROW[partner] + assert row in contract, ( + f"[{partner}] the registry has no `[contract.{row}]` row. That row is the " + "authority this package's CONSUMER_DOCUMENT_NAME mirrors (ADR-017 §5); without " + "it there is nothing to check the mirror against. Either it was retired upstream " + "or this file names the wrong row." ) - assert found[0] == _product(partner).CONSUMER_DOCUMENT_NAME, ( - "the declared pin in this file and the module's constant disagree — the " - "always-on test above should have caught this first" + declared = contract[row].get("value") + ours = _product(partner).CONSUMER_DOCUMENT_NAME + assert declared == ours, ( + f"[{partner}] this package writes every delivery under {ours!r}, but the platform " + f"registry declares {declared!r} in `[contract.{row}]`. The registry is the " + "authority — the consumer owns this name and changing it is a contract amendment. " + "A delivery under a name the consumer does not ask for is uploaded, stored, and " + "invisible (ADR-013 §4.1a)." ) - - manager = (pkg / "managers" / "prediction" / "manager.py").read_text() - assert _CONSUMER_FILTER in manager, ( - f"{CONSUMER_REPO[partner]} no longer selects by " - f"{_CONSUMER_FILTER!r}. The name may still match while the consumer filters on " - "something else entirely — same invisibility, different cause. Re-read its " - "selection path before assuming this repository's deliveries are reachable." + assert declared == _CONSUMER_DOCUMENT_NAME[partner], ( + f"[{partner}] the pin in this file disagrees with the registry — the local " + "always-on test above should have caught a mismatch with the module first." ) diff --git a/tests/test_register_integrity.py b/tests/test_register_integrity.py index bdde1ad..e8b4491 100644 --- a/tests/test_register_integrity.py +++ b/tests/test_register_integrity.py @@ -186,6 +186,56 @@ def test_internal_references_resolve_and_foreign_ones_are_namespaced(register): ) +def test_test_files_named_by_live_entries_exist_or_name_their_repo(register): + """A live entry pointing at a test nobody can find describes work against nothing. + + Scoped to **Open Concerns and Disagreements** deliberately. A resolved entry citing + `test_mapping.py` or `test_reconciliation_parity.py` is recording what discharged it, + and those files are correctly gone — a blanket existence check would fire on eleven + such mentions and be deleted within a day (ADR-014 §3). Measured before scoping: + eleven missing across the whole register, **two** in live entries. + + The two it found on 2026-08-05 were both real, and one of them is a shape this + register already polices for identifiers but not for paths: + ``tests/forecast/test_wire_golden_fixture.py`` is **views-faoapi's** file. A bare + foreign path sends the reader hunting in this repo's tree, which is exactly the + argument `test_internal_references_resolve_and_foreign_ones_are_namespaced` makes + about a bare ``C-161``. + + **The exemption is a declaration, not a proximity heuristic, and the first draft got + that wrong.** It reused `_FOREIGN_PREFIXES` — the identifier-namespacing list — over a + 60-character window. That list holds ordinary English: ``models``, ``frames``, + ``pipeline-core``. Mutation M1 planted a vanished file in a live entry and the guard + stayed green, because the sentence three words earlier happened to say *"pinned + pipeline-core-free"*. It had caught its two real findings by luck of their neighbours + and would have missed most others. So the owning repo must now be named **immediately + before the path**, in the possessive form a reader would write anyway. + """ + body = register.split("## Register Conventions")[0] + live = body[body.index("## Open Concerns"):body.index("## Resolved Concerns")] + + repo = Path(__file__).resolve().parent.parent + #: `views-faoapi's `tests/...`` — the repo abutting the path, not merely nearby. + owned_elsewhere = re.compile(r"views-[\w-]+(?:'s)?[\s:]*$") + + unresolved = [] + for match in re.finditer(r"`(?:(tests/[\w/]+\.py)|(test_\w+\.py))`", live): + rel = match.group(1) or f"tests/{match.group(2)}" + if (repo / rel).exists(): + continue + if owned_elsewhere.search(live[max(0, match.start() - 40) : match.start()]): + continue + line = live.count("\n", 0, match.start()) + 1 + unresolved.append(f"line ~{line} of the live sections: {rel}") + + assert not unresolved, ( + "live register entries name test files that do not exist here and do not name " + f"the repository that owns them: {unresolved}. If the file is another repo's, " + "say so beside it. If it is ours, it was deleted and the entry is describing " + "work against a file nobody can open." + ) + + # ── Closing conditions must not already be met (S2 / #183) ─────────────────── # # The guards above catch a heading that SAYS it is resolved. They cannot catch an diff --git a/tests/test_store_construction.py b/tests/test_store_construction.py new file mode 100644 index 0000000..1abd151 --- /dev/null +++ b/tests/test_store_construction.py @@ -0,0 +1,173 @@ +"""Store construction, called without a manager — register C-40's consequence (a). + +C-40's first stated consequence is that *"the FAO logic cannot be instantiated or +unit-tested without the full framework + Appwrite env + viewser."* For the store +construction that was literally true until 2026-08-05: building the production-forecasts +datastore meant calling ``self._prod_forecasts_datastore()``, which meant having a +manager, which meant a views-models path manager and a live environment. + +Nothing about that code needed any of it. It read one value out of ``self.configs`` and +assigned a local to ``self.ensemble_path_manager`` that nothing ever read back. Turning +the three store builders into module-level functions with declared arguments removed the +requirement entirely — and this file is the proof, because every test here runs with no +manager instance and with the Appwrite variables explicitly *removed* from the +environment. + +**What this does not test.** That a real store connects, or that the credentials work. +That needs the production Appwrite project, which þing-02 D2 forbids testing against and +which is the only project that exists (A3(h), answered 2026-08-05). What is testable +offline is the part that was previously untestable at any price: the refusals, their +ordering, and the fact that the environment contract is checked *before* anything is +constructed. +""" + +from __future__ import annotations + +import pytest + +from tests.conftest import PARTNER_PACKAGES + +#: Every Appwrite name any partner's builders read. Cleared before each test so a +#: maintainer's populated shell cannot make these pass for the wrong reason — the +#: failure mode a laptop-only run has already produced twice in this repo (C-81). +_APPWRITE_PREFIX = "APPWRITE_" + + +@pytest.fixture +def no_appwrite_env(monkeypatch): + """A deliberately empty Appwrite environment.""" + import os + + for name in [k for k in os.environ if k.startswith(_APPWRITE_PREFIX)]: + monkeypatch.delenv(name, raising=False) + + +def _managers(partner: str): + pytest.importorskip( + "views_pipeline_core", + reason="the builders construct pipeline-core objects; it is a declared " + "dependency, so this skip means a broken environment rather than a valid one", + ) + return __import__( + f"views_postprocessing.{partner}.managers.{partner}", + fromlist=["_build_prod_forecasts_store"], + ) + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_the_builders_are_functions_not_methods(partner): + """The structural claim C-40's fix rests on. Assert it, do not assume it. + + If these ever migrate back onto the class, every test below would still pass while + silently requiring a manager again — the guard has to check the shape, not just the + behaviour. + """ + import inspect + + module = _managers(partner) + for name in ("_build_prod_forecasts_store", "_build_partner_store", + "_partner_appwrite_config"): + fn = getattr(module, name, None) + assert fn is not None, ( + f"[{partner}] {name} is gone from the module namespace. If it moved back " + "onto the manager class, C-40's consequence (a) is back with it." + ) + assert inspect.isfunction(fn), f"[{partner}] {name} is not a module-level function" + params = list(inspect.signature(fn).parameters) + assert "self" not in params, ( + f"[{partner}] {name} takes `self` — it is a method wearing a function's " + "name, and still cannot be called without a manager." + ) + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_a_missing_ensemble_name_is_refused_before_anything_is_built( + partner, no_appwrite_env +): + """The launch-config error, raised with no manager and no environment. + + This is the whole point of the extraction in one assertion: the refusal is + observable by calling a function with an argument. + """ + module = _managers(partner) + with pytest.raises(ValueError, match="ensemble"): + module._build_prod_forecasts_store(None) + + +@pytest.mark.parametrize("bad", ["", None], ids=["empty-string", "none"]) +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_an_empty_ensemble_name_is_refused_too(partner, bad, no_appwrite_env): + """`""` and `None` are both "not configured", and both used to reach `configs.get`. + + A launch config with `ensemble: ""` is a plausible typo and must not be treated as + a name — `EnsemblePathManager("")` would resolve to something, and what it resolves + to is not this delivery's ensemble. + """ + module = _managers(partner) + with pytest.raises(ValueError, match="ensemble"): + module._build_prod_forecasts_store(bad) + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_the_environment_contract_is_checked_before_a_store_is_constructed( + partner, no_appwrite_env +): + """Ordering, not just presence — and the ordering is the fail-loud guarantee. + + ``assert_env_declared`` must run before ``AppwriteConfig``/``DatastoreModule`` are + built, so a missing coordinate produces the declared error naming what is absent + rather than whatever the client raises when handed ``None`` for an endpoint. þing-01 + #134 turned this repo's environment handling from an implicit borrow into a declared + contract; an unchecked construction path would quietly undo that. + + The error must also name the store, because two different stores read two different + variable sets and "some Appwrite variable is missing" sends an operator to the wrong + half of the environment. + """ + module = _managers(partner) + with pytest.raises(Exception) as excinfo: + module._build_prod_forecasts_store("some_ensemble") + message = str(excinfo.value) + assert "production_forecasts" in message, ( + f"[{partner}] the refusal does not name which store's environment is " + f"incomplete: {message!r}" + ) + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_the_partner_store_names_its_own_environment_when_it_refuses( + partner, no_appwrite_env +): + """The outbound store reads a different variable set from the inbound one.""" + module = _managers(partner) + with pytest.raises(Exception) as excinfo: + module._partner_appwrite_config(model_path=None) + assert "datastore" in str(excinfo.value).lower() + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_no_coordinate_value_is_baked_into_the_builders(partner): + """The repo is PUBLIC and the registry is referenced, never copied (þing-01 S6). + + The builders read every coordinate through ``os.getenv`` by declared NAME. A literal + bucket id or endpoint appearing here would be a value copy in a public repository — + the platform's original failure. The repo-wide guard covers this too; asserted at + the seam as well because this is the file where such a literal would be most + tempting to paste while debugging. + """ + import inspect + + module = _managers(partner) + source = "".join( + inspect.getsource(getattr(module, name)) + for name in ("_build_prod_forecasts_store", "_partner_appwrite_config") + ) + for line in source.splitlines(): + if "os.getenv(" in line: + assert "os.getenv(\"APPWRITE_" in line or "os.getenv('APPWRITE_" in line, ( + f"[{partner}] a coordinate is read by something other than a declared " + f"APPWRITE_ name: {line.strip()!r}" + ) + assert "https://" not in line or "#" in line.split("https://")[0], ( + f"[{partner}] a literal endpoint appears in the builders: {line.strip()!r}" + ) diff --git a/tests/test_store_port.py b/tests/test_store_port.py new file mode 100644 index 0000000..cc99094 --- /dev/null +++ b/tests/test_store_port.py @@ -0,0 +1,155 @@ +"""The store port's refusal, tested — register C-79. + +**This code had zero tests until 2026-08-05**, while the comment beside it called it +"the whole mechanism". It is: pipeline-core's store, on a metadata failure *after* the +file is uploaded, logs the error and **returns** ``success=False`` rather than raising. +A caller that discards the result therefore ships a file with no metadata document — +invisible to the consumer, which is what happened to run-0's historical artifact on +2026-07-27. This port is the thing that turns that into a refusal. + +Testable now for a reason worth stating: the standing excuse for source-scanning +manager-side facts is that the managers need Appwrite env and a views-models path +manager to instantiate. ``_ContractStorePort`` needs neither — it takes a store object +and calls four methods on it. A fake store is enough, so the excuse never applied here. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import pytest + +from tests.conftest import PARTNER_PACKAGES + + +@dataclass +class _Result: + """Shaped like pipeline-core's ``OperationResult``: a ``success`` bool plus error.""" + + success: object + error: str | None = None + + +class _FakeStore: + """Records the upload and returns whatever result the test declares.""" + + def __init__(self, result): + self.result = result + self.calls = [] + + def upload_data(self, **kwargs): + self.calls.append(kwargs) + return self.result + + +def _port(partner: str, result): + """The partner's port, wrapping a fake store. Needs no Appwrite environment.""" + pytest.importorskip("views_pipeline_core", reason="the port wraps its DatastoreModule") + module = __import__( + f"views_postprocessing.{partner}.managers.{partner}", fromlist=["_ContractStorePort"] + ) + store = _FakeStore(result) + return module._ContractStorePort(store), store + + +def _upload(port, tmp_path): + payload = tmp_path / "artifact.parquet" + payload.write_bytes(b"x") + port.upload( + payload, + filename=payload.name, + name="un_fao", + doc_type="model", + category="historical", + loa="pgm", + targets=["lr_ged_sb"], + description="{}", + ) + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_a_successful_upload_is_accepted(partner, tmp_path): + """The happy path must not raise, or the guard would block every delivery.""" + port, store = _port(partner, _Result(success=True)) + _upload(port, tmp_path) + assert len(store.calls) == 1, "the port must forward exactly one upload to the store" + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_a_reported_failure_is_refused_and_names_the_error(partner, tmp_path): + """The defect this port exists for: uploaded file, failed metadata, success=False.""" + port, _ = _port(partner, _Result(success=False, error="metadata storage failed")) + with pytest.raises(RuntimeError) as excinfo: + _upload(port, tmp_path) + message = str(excinfo.value) + assert "metadata storage failed" in message, ( + "the refusal must carry the store's own error — an operator debugging a partial " + "upload should not have to go find it" + ) + assert "orphan" in message, ( + "the refusal must say what the failure MEANS: a file with no metadata document, " + "which is invisible to the consumer rather than absent" + ) + + +@pytest.mark.parametrize( + "result, why", + [ + (None, "the store returned nothing at all"), + (_Result(success=None), "success is None — the fail-open case C-79 named"), + (_Result(success="ok"), "success is a truthy non-bool"), + (object(), "the result has no success attribute"), + ], +) +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_an_unrecognised_result_is_refused_rather_than_assumed_good( + partner, result, why, tmp_path +): + """Fail CLOSED. This is the polarity fix, and it is the whole of C-79. + + The check was ``if success is False``, so every case above sailed through as though + the upload had worked. Today ``upload_data`` has one return path and ``success`` is + a ``bool``, so the two polarities agree — but "today" is doing a lot of work in that + sentence, and this entry's own trigger is *when pipeline-core changes what + ``upload_data`` returns*. Fail-open is the wrong side to be on when the question is + whether a partner delivery actually landed. + """ + port, _ = _port(partner, result) + with pytest.raises(RuntimeError, match="did not fully succeed"): + _upload(port, tmp_path) + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_the_refusal_names_what_it_actually_got(partner, tmp_path): + """A refusal an operator cannot act on is barely better than no refusal. + + ``success=None`` and ``success=False`` are different faults — one is a store that + reported a failure, the other a store whose contract moved. The message has to + separate them or the next person re-runs the delivery instead of reading a changelog. + """ + port, _ = _port(partner, _Result(success=None)) + with pytest.raises(RuntimeError) as excinfo: + _upload(port, tmp_path) + assert "success=None" in str(excinfo.value) + + +@pytest.mark.parametrize("partner", PARTNER_PACKAGES) +def test_the_port_forwards_every_declared_field(partner, tmp_path): + """The port is a four-method seam; dropping a field here loses it silently. + + ``name`` in particular is what the consumer filters on (register C-77) — an upload + that arrives without it is stored, billed, and unretrievable. + """ + port, store = _port(partner, _Result(success=True)) + _upload(port, tmp_path) + forwarded = store.calls[0] + # Note `doc_type` -> `type`: the port renames it on the way through, because + # `type` is the store's field name and a builtin here. That rename is the kind of + # thing a seam quietly loses, which is why the field list is asserted rather than + # assumed. + for field in ("file", "filename", "name", "type", "category", "loa", "targets", + "description"): + assert field in forwarded, f"the port dropped {field!r} on its way to the store" + assert forwarded["name"] == "un_fao" + assert forwarded["category"] == "historical" + assert forwarded["type"] == "model", "doc_type must arrive as the store's `type`" diff --git a/views_postprocessing/contract/enrichment.py b/views_postprocessing/contract/enrichment.py deleted file mode 100644 index 75baa77..0000000 --- a/views_postprocessing/contract/enrichment.py +++ /dev/null @@ -1,317 +0,0 @@ -"""Lookup-based geographic enrichment (ADR-011). - -Drop-in replacement for the runtime spatial mapper's -``enrich_dataframe_with_pg_info``. Instead of loading 774 MB of shapefiles and -computing spatial intersections at run time, it merges a precomputed lookup -table (built by ``scripts/build_gaul_lookup.py`` from the views-datafactory's -area-majority GAUL parquets) onto the input DataFrame by PRIO-GRID cell id. - -No geopandas, no shapefiles, no spatial computation. The lookup contains only -fully-complete cells; an unknown or incomplete cell id gathers to null, so the -delivery's null gate still crashes (fail-loud) rather than shipping a hole. This -is intentional and matches the old mapper's behaviour (it returned ``None`` for -such cells). - -**The lookup side is pandas-free (S4 / #89, epic #85).** It is read with pyarrow -and held as numpy arrays plus a sorted key index; attaching metadata to a frame is -a **keyed gather**, not frame algebra, and it never needed a pandas merge. The -input frame is still whatever the caller passes — this class is the *build and -verification* path's object, and its callers hand it DataFrames. What changed is -that the ~880 KiB artifact is no longer materialised as a pandas frame, and the -join no longer depends on the artifact carrying pandas index metadata — which is -what unblocks S5 (#90) making the builder pyarrow-native. - -Produces exactly the 9-column contract enforced at -the delivery's artifact builders and at -views-faoapi ``handlers.py`` (``FAO_PGMDataset._METADATA_COLS``). -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import TYPE_CHECKING - -import numpy as np -import pyarrow as pa - -from views_postprocessing.contract import gaul_lookup - -if TYPE_CHECKING: # pragma: no cover — pandas is in this module's INTERFACE, not its - import pandas as pd # implementation. Callers hand it DataFrames; nothing here - # constructs, reads or joins one. Removing the runtime import is the point of - # S4 (#89): the ~880 KiB lookup is no longer materialised as a pandas frame, and the - # join no longer needs the artifact to carry pandas index metadata. -from views_postprocessing.contract.gaul_schema import METADATA_COLS - -logger = logging.getLogger(__name__) - -# The artifact's identity lives in `gaul_lookup` (#152, C-68) — this alias keeps the -# enricher's own default working without re-deriving the path. -_DEFAULT_LOOKUP = gaul_lookup.LOOKUP_PATH - -#: int64 bounds, named because they are a correctness condition rather than trivia: -#: a value outside them does not raise on cast, it WRAPS, and a wrapped id that is -#: flagged valid is the fabricated value this module forbids. -_INT64_MIN = int(np.iinfo(np.int64).min) -_INT64_MAX = int(np.iinfo(np.int64).max) - -#: Float-comparison bounds. Deliberately NOT the int bounds above: 2**63-1 is odd and -#: unrepresentable in float64, so comparing a float against it rounds up to 2**63 and -#: lets through the very values the bound excludes. 2**63 and -2**63 are both powers -#: of two and exact, so the float check is `>= -2**63` and `< 2**63`. -_INT64_MIN_F = -(2.0**63) -_INT64_MAX_EXCLUSIVE_F = 2.0**63 - -#: The lookup's key column. Named once — the artifact calls it `priogrid_gid`, the -#: wire calls it `priogrid_id` (§5.1), and confusing the two is a silent join failure. -_KEY = "priogrid_gid" - - -class GaulLookupEnricher: - """Merge precomputed GAUL metadata onto an input DataFrame by cell id.""" - - def __init__(self, lookup_path: str | Path | None = None) -> None: - self._lookup_path = Path(lookup_path) if lookup_path else _DEFAULT_LOOKUP - if not self._lookup_path.exists(): - err_msg = ( - f"GAUL lookup table not found at {self._lookup_path}. " - f"Build it with scripts/build_gaul_lookup.py." - ) - logger.error(err_msg) - raise FileNotFoundError(err_msg) - table = gaul_lookup.load(self._lookup_path) - missing = [c for c in (_KEY, *METADATA_COLS) if c not in table.column_names] - if missing: - err_msg = f"Lookup table is missing contract columns: {missing}" - logger.error(err_msg) - raise ValueError(err_msg) - - if table.column(_KEY).null_count: - err_msg = ( - f"GAUL lookup at {self._lookup_path} has null values in {_KEY!r}. A " - "null key cannot identify a cell, and coercing it would make it " - "collide with any unusable id on the query side — the row would then " - "be reported as FOUND and receive another cell's metadata. Rebuild " - "with scripts/build_gaul_lookup.py. (That script does not check the " - "key column either — its null check runs after the key becomes the " - "index, and DataFrame.isna() does not inspect an index. A null key is " - "unreachable there only because the earlier astype('int64') raises. " - "See register C-76.)" - ) - logger.error(err_msg) # ADR-008: logged persistently AND raised - raise ValueError(err_msg) - - if table.num_rows == 0: - err_msg = ( - f"GAUL lookup at {self._lookup_path} is empty. Every cell would gather " - "to null and the delivery would fail downstream complaining about " - "missing metadata rather than about a missing lookup. Rebuild it with " - "scripts/build_gaul_lookup.py." - ) - logger.error(err_msg) # ADR-008: logged persistently AND raised - raise ValueError(err_msg) - - # Sort the whole table once by key, in arrow, so the gather below is a binary - # search per row rather than a scan. `take` reorders every column together — - # doing it column-by-column in python was measurably quadratic and is exactly - # the mistake this comment exists to stop the next person repeating. - keys = table.column(_KEY).to_numpy(zero_copy_only=False).astype(np.int64) - order = np.argsort(keys, kind="stable") - table = table.take(pa.array(order)) - self._keys = keys[order] - # One `to_pylist` per column, not per row. Lists rather than numpy arrays - # because the columns are of mixed kind (float codes, string names) and the - # output is assembled per column anyway. - # - # DTYPE CHANGE, deliberate and measured (S4 / #89). The pandas merge this - # replaced produced `category` name columns, inherited from the artifact's - # dictionary encoding; assigning lists produces `object`. On a 200-row output: - # category 1,795,191 bytes vs object 60,061 — a categorical carries the full - # 64,742-entry dictionary whatever the output size, so this is far lighter for - # the small frames this object actually sees and heavier only past the point - # where the dictionary amortises. The builder's `# names/iso categorical (C-32 - # memory)` note governs the ARTIFACT; it never governed this method's output. - self._values = {col: table.column(col).to_pylist() for col in METADATA_COLS} - self.lookup_version = self._read_version(self._lookup_path) - logger.info( - "Loaded GAUL lookup: %d cells from %s (version=%s)", - len(self._keys), self._lookup_path, self.lookup_version, - ) - - @staticmethod - def _as_cell_ids(gids) -> tuple[np.ndarray, np.ndarray]: - """``(int64 ids, usable mask)`` — DECLARED, never coerced (ADR-003). - - A cell id is an integer. Anything that is not one — a missing value, a - non-integral float, a ``pd.NA`` — is marked **unusable** rather than cast, - and unusable ids gather to null exactly as an unknown gid does. The - downstream gate then sees a hole, which is what it exists for. - - **Why not just cast.** ``np.asarray(gids, dtype=np.int64)`` looks equivalent - and is not, in three ways found in review of #210: - - - a ``NaN`` becomes ``INT64_MIN`` with only a ``RuntimeWarning``. If a lookup - key were ever null it would take the same sentinel, the two would collide, - and the row would be reported FOUND — receiving another cell's metadata. - That is the fabricated value this module's docstring forbids (cf. C-35). - - a non-integral float **truncates silently**: ``54220.000000001`` becomes - ``54220`` and matches a real, *different* cell. Verified against the pandas - merge this replaced: it returns null there, and even warns. A silent wrong - match is strictly worse than the crash it would replace. - - a ``pd.NA`` raises a bare ``TypeError`` from numpy — no log, no - contract-shaped error, unlike every other guard in this file. - """ - arr = np.asarray(gids) - - if arr.dtype.kind in ("i", "u"): - # In range, or it is not the id the caller wrote. uint64 max silently - # wraps to -1 under `astype(np.int64)` and would be flagged VALID. - usable = (arr >= _INT64_MIN) & (arr <= _INT64_MAX) - return np.where(usable, arr, 0).astype(np.int64), usable - - if arr.dtype.kind == "f": - # Finite, integral, AND representable. `1e30` passes the first two and - # then overflows the cast to INT64_MIN — an id nobody asked for, marked - # valid. Same silent-coercion class this method exists to remove. - # STRICTLY below 2**63, not `<= _INT64_MAX`. `_INT64_MAX` is 2**63-1, - # which is odd and NOT representable in float64 — comparing against it - # promotes to float and rounds UP to 2**63, so the bound admitted exactly - # the values it was added to exclude. `float(2**63)` passed every check - # and then wrapped to INT64_MIN, flagged valid. `_INT64_MIN` has no - # equivalent hole: -2**63 is a power of two and exactly representable. - # - # `isfinite` is kept for intent and is SUBSUMED — `inf` fails the upper - # bound, `NaN` fails `arr == rint(arr)` (NaN equals nothing). It is the - # one condition here that survives its own removal, so it is defensive - # rather than load-bearing; the other three each fail the suite. - usable = ( - np.isfinite(arr) - & (arr == np.rint(arr)) - & (arr >= _INT64_MIN_F) - & (arr < _INT64_MAX_EXCLUSIVE_F) - ) - return np.where(usable, arr, 0).astype(np.int64), usable - - # object / pandas-nullable. Accept only values that ALREADY ARE integers. - # - # An earlier draft used `float(value)`, which parses. That accepted the string - # `"54220"` as a cell id — where the pandas merge this replaced raised - # `ValueError: You are trying to merge on object and int64 columns`. A string - # gid column is a declaration error and the old path said so; parsing it is - # inference (ADR-003) and it is the same defect as the drifted float, just - # pointing the other way. `bool` is excluded for the same reason: `True` is - # not cell 1. - ids = np.zeros(len(arr), dtype=np.int64) - usable = np.zeros(len(arr), dtype=bool) - for i, value in enumerate(arr): - # `np.timedelta64` IS an `np.integer` instance — a numpy quirk, and - # `np.datetime64` is not, so the hole was specific to that one type. A - # duration is not a cell id any more than `True` is cell 1. - if isinstance(value, (bool, np.timedelta64)): - continue - if not isinstance(value, (int, np.integer)): - continue - if _INT64_MIN <= int(value) <= _INT64_MAX: - ids[i], usable[i] = int(value), True - return ids, usable - - def _gather(self, gids) -> dict: - """The metadata for each gid, warning about the ones that got none. - - A sorted-key ``searchsorted`` rather than a hash map: the lookup is 64,742 - rows read once per process, and the gather is over the delivery's row count. - Absent gids yield ``None`` in every column — the null the downstream gate - exists to catch, not a sentinel it would pass. An id that is not a usable - cell id is treated the same way: absent, never guessed at. - """ - wanted, usable = self._as_cell_ids(gids) - # `self._keys` is non-empty — __init__ refuses an empty lookup. An earlier draft - # guarded with `(len(self._keys) > 0) & (...)`, which READS as a guard and is - # not one: `&` evaluates both operands, so the index happened regardless and an - # empty lookup raised IndexError from inside the gather rather than ValueError - # from the constructor. Guard where the condition is knowable, not where it bites. - idx = np.clip(np.searchsorted(self._keys, wanted), 0, len(self._keys) - 1) - # `& usable` is NOT belt-and-braces, and no substitute value would make it so. - # Unusable ids substitute to 0, but int64 reserves nothing — the lookup is - # arbitrary data and may legally contain 0, or -1, or INT64_MIN, or whatever - # else one might pick instead. The mask is what carries correctness; the - # substitute is only a placeholder. Pinned by - # `test_an_unusable_id_cannot_match_the_cell_it_was_substituted_with`. - found = (self._keys[idx] == wanted) & usable - out = { - col: [vals[i] if hit else None for i, hit in zip(idx, found)] - for col, vals in self._values.items() - } - - # Warn HERE rather than handing the caller three arrays to reassemble one - # message. `__init__` already co-locates detection and logging at every guard - # (ADR-008); this is the same shape. An earlier draft returned four values, - # three of which existed only to build the string below. - absent = ~found - n_unmapped = int(absent.sum()) - if n_unmapped: - unknown = sorted({int(i) for i, miss, ok in zip(wanted, absent, usable) if miss and ok}) - n_unusable = int((absent & ~usable).sum()) - detail = f"unknown cells {unknown[:20]}" if unknown else "no unknown cells" - if n_unusable: - detail += f"; {n_unusable} row(s) carried no usable cell id" - logger.warning( - "%d/%d rows have no lookup match (will fail validation): %s", - n_unmapped, len(wanted), detail, - ) - return out - - @staticmethod - def _read_version(path: Path) -> str: - """The lookup's build stamp. Delegates to ``gaul_lookup.version`` (#152) — - this was a ``@staticmethod`` that never touched the instance, i.e. a fact - about the artifact, not about the enricher.""" - return gaul_lookup.version(path) - - def enrich_dataframe_with_pg_info( - self, - df: pd.DataFrame, - pg_id_col: str = "priogrid_gid", - time_id_col: str = "month_id", - only_metadata: bool = True, - **ignored_mapper_kwargs, - ) -> pd.DataFrame: - """Return ``df`` with the 9 metadata columns merged in by cell id. - - Signature mirrors the mapper's method so the manager call site changes - minimally. Mapper-only kwargs (``batch_size``, ``use_multiprocessing``, - ``show_progress`` …) are accepted and ignored — a table join needs none - of them — but any unrecognised kwarg is logged at debug so a genuine - caller mistake is not wholly silent. - - Cells absent from the lookup get NaN metadata (fail-loud downstream). - """ - if ignored_mapper_kwargs: - logger.debug( - "GaulLookupEnricher ignoring mapper-only kwargs: %s", - sorted(ignored_mapper_kwargs), - ) - if pg_id_col not in df.columns: - err_msg = f"Column '{pg_id_col}' not found in DataFrame" - logger.error(err_msg) - raise ValueError(err_msg) - - if only_metadata: - keep = [pg_id_col] - if time_id_col in df.columns: - keep.append(time_id_col) - base = df[keep].copy() - else: - base = df.copy() - - gids = base[pg_id_col].to_numpy() - gathered = self._gather(gids) - - merged = base.copy() - for col in METADATA_COLS: - merged[col] = gathered[col] - return merged - - # Convenience alias for new call sites that don't need the legacy name. - enrich = enrich_dataframe_with_pg_info diff --git a/views_postprocessing/contract/frame_extraction.py b/views_postprocessing/contract/frame_extraction.py index ad266fa..2241fbf 100644 --- a/views_postprocessing/contract/frame_extraction.py +++ b/views_postprocessing/contract/frame_extraction.py @@ -1,24 +1,21 @@ """Frame-native representation seam: extract primitives from a views-frames frame. -The **frame counterpart** to ``extraction.py`` (the pandas seam). It returns the *same* -primitives — sets of ints, numpy month arrays — so the representation-free +Returns primitives — sets of ints, numpy month arrays — so the representation-free ``views_postprocessing.delivery`` invariants consume them unchanged. -Per the migration design (epic #85): pandas and views-frames do **not** coexist at runtime, -so these are deliberately **siblings** of the pandas readers in ``extraction.py``, not a -replacement, and there is **no shared ``Extractor`` Protocol** (a polymorphic interface -nobody dispatches on would be speculative — YAGNI/ISP). When the forecast interior moves to a -frame (S3 / #88), the manager calls *these*; the pandas readers stay for the still-pandas -historical path (gated on C-40 / S7). - -Scope: the readers the forecast interior needs — distinct cells and months from the frame's -index. Deliberately **not** here yet (no speculative code): -- the pandas→``(N, S)`` sample-array unpacker — added when rusty_bucket (#143) declares the - layout (the seam will be *told* the layout, never sniff it); -- a frame-native ``unmapped_cell_count`` — geographic metadata lives on the pandas enriched - frame, not the value frame, until the enrichment moves off pandas (S4 / #89); -- a frame-native ``drop_months_above`` — the observed-range clip is on the *historical* - frame, which is gated on the inbound retirement (S7 / #92). +**This module was one of a pair and is now alone.** It was written as the frame-native +sibling of a pandas seam (``extraction.py``), deliberately duplicated rather than +abstracted while both representations were live: pandas and views-frames did not coexist +at runtime, and a shared ``Extractor`` Protocol nobody dispatched on would have been +speculative. That migration is finished — the pandas seam was deleted in #151, and #90 +retired the last module holding a pandas import at all. There is no sibling and no +still-pandas path; what remains is this one seam, and the docstring is corrected rather +than left describing a pair. + +Scope: the readers the forecast interior needs — distinct cells and months from the +frame's index. Deliberately **not** here (no speculative code): a sample-array unpacker, +added when the producer declares its layout — the seam will be *told* the layout, never +sniff it (ADR-003). """ from __future__ import annotations diff --git a/views_postprocessing/contract/gaul_lookup.py b/views_postprocessing/contract/gaul_lookup.py index 23f17bb..7090812 100644 --- a/views_postprocessing/contract/gaul_lookup.py +++ b/views_postprocessing/contract/gaul_lookup.py @@ -7,11 +7,10 @@ times per run** (C-66) — once eagerly into a pandas enricher it never used, then twice more through pyarrow. -The artifact's identity does not belong to the enricher. ``GaulLookupEnricher`` is -one *consumer* of this asset (the pandas merge used by the build/verification path); -the contract delivery is another, and it wants the table and the stamp, not the -merge. Splitting them lets the delivery read the file once, in arrow, with no pandas -anywhere on the path. +The artifact's identity never belonged to that class, which is why splitting them was +the fix: the delivery wants the table and the stamp, not a merge. It reads the file +**once**, in arrow, with no pandas anywhere on the path. The enricher itself was retired +in #90 (register **C-75**) once it was established it had no production caller. Verified by ``tests/test_gaul_lookup_fidelity.py``: the committed artifact matches views-datafactory's authoritative GAUL parquets value-for-value, its key is unique, diff --git a/views_postprocessing/contract/gaul_schema.py b/views_postprocessing/contract/gaul_schema.py index c2b9513..1a49e49 100644 --- a/views_postprocessing/contract/gaul_schema.py +++ b/views_postprocessing/contract/gaul_schema.py @@ -1,6 +1,6 @@ """Single source of truth for the GAUL enrichment schema and PRIO-GRID geometry. -Shared by the production enricher (enrichment.py) and the build/diff tooling +Shared by the delivery's artifact builders and the build/diff tooling (scripts/) so the 9-column contract, the datafactory->contract rename map, and the PRIO-GRID coordinate formula are defined exactly once. diff --git a/views_postprocessing/contract/launch_config.py b/views_postprocessing/contract/launch_config.py index 2b88f56..8ebcafb 100644 --- a/views_postprocessing/contract/launch_config.py +++ b/views_postprocessing/contract/launch_config.py @@ -65,6 +65,46 @@ def assert_contract_mode(configs: dict) -> None: raise LaunchConfigError(err_msg) +def assert_queryset_was_importable(queryset: object | None) -> None: + """Raise if the queryset could not be read at all — before asking what it declares. + + **Register C-83: three correct behaviours composing into a lie.** pipeline-core's + ``ModelPathManager.get_queryset()`` catches any exception from importing + ``config_queryset.py``, logs it, and returns ``None``. ``declared_data_format(None)`` + then returns ``'dataframe'`` — the documented default for a non-dict. And + ``assert_frame_native_historical('dataframe')`` says: + + the queryset declares data_format='dataframe' … Set data_format: + 'feature_frame' in the postprocessor's config_queryset + + ...pointing at a file that already says ``feature_frame``. A queryset that **failed + to import** was indistinguishable from one that **declared the wrong format**, and + the message arrived on the live FAO path while someone was fixing a failed run. + + Each layer is faithful on its own. What goes wrong is that the *absence* of an + answer is silently given the shape of an answer — ADR-003's rule broken by + composition rather than by anyone inferring anything. + + Args: + queryset: the return of ``model_path.get_queryset()``. + + Raises: + LaunchConfigError: saying the queryset could not be imported, which is a + different problem from a queryset that declares the wrong thing. + """ + if queryset is None: + err_msg = ( + "the postprocessor's config_queryset could not be imported — " + "`get_queryset()` returned None, which it does for ANY exception raised " + "while importing that module (pipeline-core swallows it and logs). This is " + "NOT a declaration problem: do not edit data_format until the module " + "imports. Check the traceback pipeline-core logged just above this, and a " + "missing sibling checkout or dependency first." + ) + logger.error(err_msg) # ADR-008: logged persistently AND raised + raise LaunchConfigError(err_msg) + + def assert_frame_native_historical(data_format: str | None) -> None: """Raise unless the queryset descriptor declares the frame-native historical read. diff --git a/views_postprocessing/crafd/appwrite_env.py b/views_postprocessing/crafd/appwrite_env.py index 68c42ac..fcf2917 100644 --- a/views_postprocessing/crafd/appwrite_env.py +++ b/views_postprocessing/crafd/appwrite_env.py @@ -6,9 +6,9 @@ views-appwrite and referenced by pinned URL, never copied — copies were the platform's original failure (þing-01 sáttmál S6): - https://github.com/views-platform/views-appwrite/blob/90fc105/docs/ADRs/platform/coordinate_registry.toml + https://github.com/views-platform/views-appwrite/blob/c7b597e/docs/ADRs/platform/coordinate_registry.toml -That pin is registry **v1.4.1** — declared below as ``SEAM_CONTRACT_VERSION`` / +That pin is registry **v1.5.2** — declared below as ``SEAM_CONTRACT_VERSION`` / ``SEAM_CONTRACT_COMMIT`` so the pin is a value a test can check rather than a fact buried in prose. A pinned URL does not rot, but it does go stale, and nothing in this repository could previously tell you it had (register C-57). @@ -47,8 +47,8 @@ #: declaration against that edition of the registry. ``tests/test_env_declaration.py`` #: enforces the pair — for this package and for ``unfao`` alike — against a local #: views-appwrite checkout when one is present. -SEAM_CONTRACT_VERSION = "1.4.1" -SEAM_CONTRACT_COMMIT = "90fc105" +SEAM_CONTRACT_VERSION = "1.5.2" +SEAM_CONTRACT_COMMIT = "c7b597e" CONNECTION_ENV = ( "APPWRITE_ENDPOINT", diff --git a/views_postprocessing/crafd/managers/crafd.py b/views_postprocessing/crafd/managers/crafd.py index 12119f5..903e47b 100644 --- a/views_postprocessing/crafd/managers/crafd.py +++ b/views_postprocessing/crafd/managers/crafd.py @@ -56,19 +56,116 @@ def upload(self, file_path, *, filename, name, doc_type, category, loa, targets, # the claim; its line number moves between releases). It never raises, so a # caller that discards the result ships an invisible orphan: run-0's historical # artifact, 2026-07-27. This check is the whole mechanism. - # (Until 2026-08-03 this comment said the store "only LOGS": false, and - # self-defeating — if it only logged, `success` would be True.) + # + # **Refuse unless success is explicitly True** (register C-79). The earlier + # `if success is False` failed OPEN: a result that was None, or lacked the + # attribute, or carried a non-bool, sailed through as though the upload had + # worked. Today `upload_data` has a single return path and `success` is a + # `bool` dataclass field, so the two polarities agree — but the moment that + # stops being true is exactly this entry's trigger, and fail-open is the wrong + # side to be on when the subject is "did the delivery actually land". + # + # The old `to_dict()` fallback is gone with it: dead on the real path, and an + # unrecognised result should be refused and named, not adapted to silently. success = getattr(result, "success", None) - if success is None and hasattr(result, "to_dict"): - success = result.to_dict().get("success") - if success is False: + if success is not True: error = getattr(result, "error", None) or "unknown store error" raise RuntimeError( - f"upload of {filename!r} did not fully succeed (file may be an " - f"orphan without a metadata document): {error}" + f"upload of {filename!r} did not fully succeed (file may be an orphan " + f"without a metadata document): {error}. The store reported " + f"success={success!r} (result type {type(result).__name__})." ) +def _build_prod_forecasts_store(ensemble_name: str | None) -> DatastoreModule: + """The shared internal store (ADR-013's "shared shelf"), built from the + launcher-assembled environment (validated fail-loud; þing-01 #134 — no dotenv is + loaded here). + + pipeline-core's ``DatastoreModule.get_predictions_by_metadata`` injects an automatic + ``name == model_name`` filter on every lookup. The contract read must **not** have + it: ADR-013 artifacts are named ``{run_id}__{target}__m{month}.arrow.parquet`` + (never the bare ensemble name), so an injected ``name == "rusty_bucket"`` matches + nothing and also clobbers the wire layer's own run-id / target / name filters. + Suppressed unconditionally below — the retired legacy reader was the only caller + that needed it on (#149). + + **A function, not a method (register C-40).** It reads no manager state: the + ensemble name arrives as an argument rather than through ``self.configs``, and the + ``EnsemblePathManager`` is local — it used to be assigned to + ``self.ensemble_path_manager`` and then read exactly once, four lines later, in this + same body. Nothing else in either partner package, or in views-models, ever read it. + A method-local value wearing the costume of manager state. + + The gain is C-40's consequence (a): this is callable, and its refusals observable, + with **no manager instance, no views-models path manager and no Appwrite + environment** — see ``tests/test_store_construction.py``. + """ + if not ensemble_name: + err_msg = "Ensemble name must be provided in configs with the `ensemble` key for forecasting. Cannot proceed." + logger.error(err_msg) + raise ValueError(err_msg) + path_manager = EnsemblePathManager(ensemble_name_or_path=ensemble_name, validate=False) + + appwrite_env.assert_env_declared( + appwrite_env.CONNECTION_ENV + appwrite_env.PROD_FORECASTS_ENV, + store="production_forecasts datastore", + ) + appwrite_config = AppwriteConfig( + path_manager=path_manager, + endpoint=os.getenv("APPWRITE_ENDPOINT"), + project_id=os.getenv("APPWRITE_DATASTORE_PROJECT_ID"), + credentials=os.getenv("APPWRITE_DATASTORE_API_KEY"), + auth_method="api_key", + cache_ttl_hours=24, + bucket_id=os.getenv("APPWRITE_PROD_FORECASTS_BUCKET_ID"), + bucket_name=os.getenv("APPWRITE_PROD_FORECASTS_BUCKET_NAME"), + collection_id=os.getenv("APPWRITE_PROD_FORECASTS_COLLECTION_ID"), + collection_name=os.getenv("APPWRITE_PROD_FORECASTS_COLLECTION_NAME"), + database_id=os.getenv("APPWRITE_METADATA_DATABASE_ID"), + database_name=os.getenv("APPWRITE_METADATA_DATABASE_NAME"), + ) + datastore = DatastoreModule(appwrite_file_manager_config=appwrite_config) + # Suppress the automatic name==model_name filter (see docstring). model_path is used + # by DatastoreModule only for that injection and for uploads; the contract read + # neither uploads nor performs any model-scoped lookup. + datastore.model_path = None + return datastore + + +def _build_partner_store(model_path) -> DatastoreModule: + """The partner-facing store (`crafd_bucket`) — a function for the same reason as above. + + ``model_path`` is the framework's own path manager, so it is passed in rather than + reached for. That is the whole difference between this and a method, and it is what + lets the environment contract be checked without standing up a manager. + """ + return DatastoreModule(appwrite_file_manager_config=_partner_appwrite_config(model_path)) + + +def _partner_appwrite_config(model_path) -> AppwriteConfig: + """Env → ``AppwriteConfig`` for the partner bucket. Declared names only; the values + live in the environment and are never read into this repository's source.""" + appwrite_env.assert_env_declared( + appwrite_env.CONNECTION_ENV + appwrite_env.CRAFD_ENV, + store="crafd_bucket datastore", + ) + return AppwriteConfig( + path_manager=model_path, + endpoint=os.getenv("APPWRITE_ENDPOINT"), + project_id=os.getenv("APPWRITE_DATASTORE_PROJECT_ID"), + credentials=os.getenv("APPWRITE_DATASTORE_API_KEY"), + auth_method="api_key", + cache_ttl_hours=24, + bucket_id=os.getenv("APPWRITE_CRAFD_BUCKET_ID"), + bucket_name=os.getenv("APPWRITE_CRAFD_BUCKET_NAME"), + collection_id=os.getenv("APPWRITE_CRAFD_COLLECTION_ID"), + collection_name=os.getenv("APPWRITE_CRAFD_COLLECTION_NAME"), + database_id=os.getenv("APPWRITE_METADATA_DATABASE_ID"), + database_name=os.getenv("APPWRITE_METADATA_DATABASE_NAME"), + ) + + class CRAFDPostProcessorManager(PostprocessorManager, ForecastingModelManager): def __init__( self, @@ -82,7 +179,6 @@ def __init__( logger.info(f"Initializing {self.__class__.__name__}") self._forecast_resolution = None # {target: TargetLease}, set by _read self._historical_frame = None # views_frames.FeatureFrame, set by _read - self.ensemble_path_manager = None def _read_historical_frame(self): """#126: historical actuals as a views_frames.FeatureFrame — the first @@ -118,67 +214,16 @@ def _read_historical_data(self): """ # Declaration first: the check reads the queryset, not the loader, so a # refused config must not pay for loader construction. - launch_config.assert_frame_native_historical( - declared_data_format(self._model_path.get_queryset()) - ) + # + # Read ONCE, and distinguish "could not import it" from "it declares the wrong + # thing" (register C-83). Passing None straight into `declared_data_format` + # turns a failed import into a confident, wrong claim about the config. + queryset = self._model_path.get_queryset() + launch_config.assert_queryset_was_importable(queryset) + launch_config.assert_frame_native_historical(declared_data_format(queryset)) self._initialize_data_loader() self._read_historical_frame() - def _prod_forecasts_datastore(self) -> DatastoreModule: - """The shared internal store (ADR-013's 'shared shelf'), configured from the - launcher-assembled environment (validated fail-loud; þing-01 #134 — no dotenv - is loaded here). - - pipeline-core's ``DatastoreModule.get_predictions_by_metadata`` injects an - automatic ``name == model_name`` filter on every lookup. The contract read - must **not** have it: ADR-013 artifacts are named - ``{run_id}__{target}__m{month}.arrow.parquet`` (never the bare ensemble - name), so an injected ``name == "rusty_bucket"`` matches nothing and also - clobbers the wire layer's own run-id / target / name filters. Suppressed - unconditionally below — the retired legacy reader was the only caller that - needed it on (#149).""" - ensemble_name = self.configs.get("ensemble", None) - if not ensemble_name: - err_msg = "Ensemble name must be provided in configs with the `ensemble` key for forecasting. Cannot proceed." - logger.error(err_msg) - raise ValueError(err_msg) - self.ensemble_path_manager = EnsemblePathManager(ensemble_name_or_path=ensemble_name, validate=False) - # ensemble_configs = EnsembleManager( - # ensemble_path=self.ensemble_path_manager, - # ).configs - - # loa = ensemble_configs.get("level", None) - loa = "pgm" - if not loa: - err_msg = "level must be defined in the ensemble configurations (e.g, pgm, cm). Cannot proceed." - logger.error(err_msg) - raise ValueError(err_msg) - - appwrite_env.assert_env_declared( - appwrite_env.CONNECTION_ENV + appwrite_env.PROD_FORECASTS_ENV, - store="production_forecasts datastore", - ) - appwrite_config = AppwriteConfig( - path_manager=self.ensemble_path_manager, - endpoint=os.getenv("APPWRITE_ENDPOINT"), - project_id=os.getenv("APPWRITE_DATASTORE_PROJECT_ID"), - credentials=os.getenv("APPWRITE_DATASTORE_API_KEY"), - auth_method="api_key", - cache_ttl_hours=24, - bucket_id=os.getenv("APPWRITE_PROD_FORECASTS_BUCKET_ID"), - bucket_name=os.getenv("APPWRITE_PROD_FORECASTS_BUCKET_NAME"), - collection_id=os.getenv("APPWRITE_PROD_FORECASTS_COLLECTION_ID"), - collection_name=os.getenv("APPWRITE_PROD_FORECASTS_COLLECTION_NAME"), - database_id=os.getenv("APPWRITE_METADATA_DATABASE_ID"), - database_name=os.getenv("APPWRITE_METADATA_DATABASE_NAME"), - ) - datastore = DatastoreModule(appwrite_file_manager_config=appwrite_config) - # Suppress the automatic name==model_name filter (see docstring). model_path - # is used by DatastoreModule only for that injection and for uploads; the - # contract read neither uploads nor performs any model-scoped lookup. - datastore.model_path = None - return datastore - def _read_forecast_data_contract(self): """ADR-013 contract inbound (epic #105; streaming since the run-0 OOM fix): RESOLVE the newest fully-manifested run — manifests + pinned shard @@ -187,7 +232,7 @@ def _read_forecast_data_contract(self): released). The `wire/` package owns the policy; this method only adapts the store (DIP) and declares the product facts (region curation + coverage expectations live in the lease, where frames exist).""" - port = _ContractStorePort(self._prod_forecasts_datastore()) + port = _ContractStorePort(_build_prod_forecasts_store(self.configs.get("ensemble"))) region = self.configs.get("region") self._forecast_resolution = source_selection.resolve_run( port, @@ -307,7 +352,7 @@ def _save_contract(self) -> dict: # file itself. lookup = gaul_lookup.load() upload_enabled = bool(self.configs.get("wire_upload_enabled", product.UPLOAD_ENABLED)) - store = _ContractStorePort(self._crafd_datastore()) if upload_enabled else None + store = _ContractStorePort(_build_partner_store(self._model_path)) if upload_enabled else None # The wire is partner-neutral (#153): the manager supplies CRAF'd's product # facts explicitly rather than the mechanism reaching for them. summary = wire_sink.deliver_run( @@ -333,7 +378,15 @@ def _save_contract(self) -> dict: store.upload( hist_path, filename=hist_path.name, - name=self._model_path.model_name, + # The DECLARED consumer name, not `self._model_path.model_name` + # (register C-77). Both resolve to the same string today, because + # `model_name` is the views-models directory name and that directory + # happens to match — but only one of them is a declaration. The other + # is a filesystem coincidence in a different repository, and a + # directory rename there would strand this artifact silently: the + # consumer filters on the declared name, finds nothing, and reports + # an empty endpoint rather than an error (ADR-013 §4.1a). + name=product.CONSUMER_DOCUMENT_NAME, doc_type="model", category="historical", loa="pgm", @@ -349,29 +402,6 @@ def _save_contract(self) -> dict: summary["historical"] = hist_path.name return summary - def _crafd_datastore(self) -> DatastoreModule: - """The CRAF'd-facing store (`crafd_bucket`).""" - return DatastoreModule(appwrite_file_manager_config=self._crafd_appwrite_config()) - - def _crafd_appwrite_config(self) -> AppwriteConfig: - appwrite_env.assert_env_declared( - appwrite_env.CONNECTION_ENV + appwrite_env.CRAFD_ENV, store="crafd_bucket datastore" - ) - return AppwriteConfig( - path_manager=self._model_path, - endpoint=os.getenv("APPWRITE_ENDPOINT"), - project_id=os.getenv("APPWRITE_DATASTORE_PROJECT_ID"), - credentials=os.getenv("APPWRITE_DATASTORE_API_KEY"), - auth_method="api_key", - cache_ttl_hours=24, - bucket_id=os.getenv("APPWRITE_CRAFD_BUCKET_ID"), - bucket_name=os.getenv("APPWRITE_CRAFD_BUCKET_NAME"), - collection_id=os.getenv("APPWRITE_CRAFD_COLLECTION_ID"), - collection_name=os.getenv("APPWRITE_CRAFD_COLLECTION_NAME"), - database_id=os.getenv("APPWRITE_METADATA_DATABASE_ID"), - database_name=os.getenv("APPWRITE_METADATA_DATABASE_NAME"), - ) - def _save(self) -> dict: """Deliver the run — ADR-013 contract only (#149).""" return self._save_contract() diff --git a/views_postprocessing/unfao/appwrite_env.py b/views_postprocessing/unfao/appwrite_env.py index 1035b64..3e7122d 100644 --- a/views_postprocessing/unfao/appwrite_env.py +++ b/views_postprocessing/unfao/appwrite_env.py @@ -5,9 +5,9 @@ views-appwrite and referenced by pinned URL, never copied — copies were the platform's original failure (þing-01 sáttmál S6): - https://github.com/views-platform/views-appwrite/blob/90fc105/docs/ADRs/platform/coordinate_registry.toml + https://github.com/views-platform/views-appwrite/blob/c7b597e/docs/ADRs/platform/coordinate_registry.toml -That pin is registry **v1.4.1** — declared below as +That pin is registry **v1.5.2** — declared below as ``SEAM_CONTRACT_VERSION`` / ``SEAM_CONTRACT_COMMIT`` so the pin is a value a test can check rather than a fact buried in prose. A pinned URL does not rot, but it does go stale, and nothing in this repository could previously tell you it had (register C-57). @@ -44,8 +44,8 @@ #: Bumping these is not bookkeeping: it asserts that someone re-checked this module's #: declaration against that edition of the registry. ``tests/test_env_declaration.py`` #: enforces the pair against a local views-appwrite checkout when one is present. -SEAM_CONTRACT_VERSION = "1.4.1" -SEAM_CONTRACT_COMMIT = "90fc105" +SEAM_CONTRACT_VERSION = "1.5.2" +SEAM_CONTRACT_COMMIT = "c7b597e" CONNECTION_ENV = ( "APPWRITE_ENDPOINT", diff --git a/views_postprocessing/unfao/managers/README.md b/views_postprocessing/unfao/managers/README.md index 701f8f1..ac7d4b6 100644 --- a/views_postprocessing/unfao/managers/README.md +++ b/views_postprocessing/unfao/managers/README.md @@ -91,5 +91,5 @@ Appwrite). - [`role_and_seams.md`](../../../docs/architecture/role_and_seams.md) — role + seams - [CIC: UNFAOPostProcessorManager](../../../docs/CICs/UNFAOPostProcessorManager.md) — class contract -- [CIC: GaulLookupEnricher](../../../docs/CICs/GaulLookupEnricher.md) — the build/verification path for the GAUL lookup. **Not on this manager's call graph** (register C-75); geography is attached by `contract/historical.py` and `contract/wire/sidecar.py` +- Geography is attached by `contract/historical.py` (the actuals artifact) and `contract/wire/sidecar.py` (the §5 GAUL sidecar), each a keyed gather over the lookup this manager reads once. *(A `GaulLookupEnricher` CIC was linked here until #90; that class had no production caller and was retired — register C-75.)* - ADR-011 (mapper → lookup), ADR-012 (current ontology) diff --git a/views_postprocessing/unfao/managers/unfao.py b/views_postprocessing/unfao/managers/unfao.py index bf4e91c..e787bcd 100644 --- a/views_postprocessing/unfao/managers/unfao.py +++ b/views_postprocessing/unfao/managers/unfao.py @@ -56,19 +56,116 @@ def upload(self, file_path, *, filename, name, doc_type, category, loa, targets, # the claim; its line number moves between releases). It never raises, so a # caller that discards the result ships an invisible orphan: run-0's historical # artifact, 2026-07-27. This check is the whole mechanism. - # (Until 2026-08-03 this comment said the store "only LOGS": false, and - # self-defeating — if it only logged, `success` would be True.) + # + # **Refuse unless success is explicitly True** (register C-79). The earlier + # `if success is False` failed OPEN: a result that was None, or lacked the + # attribute, or carried a non-bool, sailed through as though the upload had + # worked. Today `upload_data` has a single return path and `success` is a + # `bool` dataclass field, so the two polarities agree — but the moment that + # stops being true is exactly this entry's trigger, and fail-open is the wrong + # side to be on when the subject is "did the delivery actually land". + # + # The old `to_dict()` fallback is gone with it: dead on the real path, and an + # unrecognised result should be refused and named, not adapted to silently. success = getattr(result, "success", None) - if success is None and hasattr(result, "to_dict"): - success = result.to_dict().get("success") - if success is False: + if success is not True: error = getattr(result, "error", None) or "unknown store error" raise RuntimeError( - f"upload of {filename!r} did not fully succeed (file may be an " - f"orphan without a metadata document): {error}" + f"upload of {filename!r} did not fully succeed (file may be an orphan " + f"without a metadata document): {error}. The store reported " + f"success={success!r} (result type {type(result).__name__})." ) +def _build_prod_forecasts_store(ensemble_name: str | None) -> DatastoreModule: + """The shared internal store (ADR-013's "shared shelf"), built from the + launcher-assembled environment (validated fail-loud; þing-01 #134 — no dotenv is + loaded here). + + pipeline-core's ``DatastoreModule.get_predictions_by_metadata`` injects an automatic + ``name == model_name`` filter on every lookup. The contract read must **not** have + it: ADR-013 artifacts are named ``{run_id}__{target}__m{month}.arrow.parquet`` + (never the bare ensemble name), so an injected ``name == "rusty_bucket"`` matches + nothing and also clobbers the wire layer's own run-id / target / name filters. + Suppressed unconditionally below — the retired legacy reader was the only caller + that needed it on (#149). + + **A function, not a method (register C-40).** It reads no manager state: the + ensemble name arrives as an argument rather than through ``self.configs``, and the + ``EnsemblePathManager`` is local — it used to be assigned to + ``self.ensemble_path_manager`` and then read exactly once, four lines later, in this + same body. Nothing else in either partner package, or in views-models, ever read it. + A method-local value wearing the costume of manager state. + + The gain is C-40's consequence (a): this is callable, and its refusals observable, + with **no manager instance, no views-models path manager and no Appwrite + environment** — see ``tests/test_store_construction.py``. + """ + if not ensemble_name: + err_msg = "Ensemble name must be provided in configs with the `ensemble` key for forecasting. Cannot proceed." + logger.error(err_msg) + raise ValueError(err_msg) + path_manager = EnsemblePathManager(ensemble_name_or_path=ensemble_name, validate=False) + + appwrite_env.assert_env_declared( + appwrite_env.CONNECTION_ENV + appwrite_env.PROD_FORECASTS_ENV, + store="production_forecasts datastore", + ) + appwrite_config = AppwriteConfig( + path_manager=path_manager, + endpoint=os.getenv("APPWRITE_ENDPOINT"), + project_id=os.getenv("APPWRITE_DATASTORE_PROJECT_ID"), + credentials=os.getenv("APPWRITE_DATASTORE_API_KEY"), + auth_method="api_key", + cache_ttl_hours=24, + bucket_id=os.getenv("APPWRITE_PROD_FORECASTS_BUCKET_ID"), + bucket_name=os.getenv("APPWRITE_PROD_FORECASTS_BUCKET_NAME"), + collection_id=os.getenv("APPWRITE_PROD_FORECASTS_COLLECTION_ID"), + collection_name=os.getenv("APPWRITE_PROD_FORECASTS_COLLECTION_NAME"), + database_id=os.getenv("APPWRITE_METADATA_DATABASE_ID"), + database_name=os.getenv("APPWRITE_METADATA_DATABASE_NAME"), + ) + datastore = DatastoreModule(appwrite_file_manager_config=appwrite_config) + # Suppress the automatic name==model_name filter (see docstring). model_path is used + # by DatastoreModule only for that injection and for uploads; the contract read + # neither uploads nor performs any model-scoped lookup. + datastore.model_path = None + return datastore + + +def _build_partner_store(model_path) -> DatastoreModule: + """The partner-facing store (`unfao_bucket`) — a function for the same reason as above. + + ``model_path`` is the framework's own path manager, so it is passed in rather than + reached for. That is the whole difference between this and a method, and it is what + lets the environment contract be checked without standing up a manager. + """ + return DatastoreModule(appwrite_file_manager_config=_partner_appwrite_config(model_path)) + + +def _partner_appwrite_config(model_path) -> AppwriteConfig: + """Env → ``AppwriteConfig`` for the partner bucket. Declared names only; the values + live in the environment and are never read into this repository's source.""" + appwrite_env.assert_env_declared( + appwrite_env.CONNECTION_ENV + appwrite_env.UNFAO_ENV, + store="unfao_bucket datastore", + ) + return AppwriteConfig( + path_manager=model_path, + endpoint=os.getenv("APPWRITE_ENDPOINT"), + project_id=os.getenv("APPWRITE_DATASTORE_PROJECT_ID"), + credentials=os.getenv("APPWRITE_DATASTORE_API_KEY"), + auth_method="api_key", + cache_ttl_hours=24, + bucket_id=os.getenv("APPWRITE_UNFAO_BUCKET_ID"), + bucket_name=os.getenv("APPWRITE_UNFAO_BUCKET_NAME"), + collection_id=os.getenv("APPWRITE_UNFAO_COLLECTION_ID"), + collection_name=os.getenv("APPWRITE_UNFAO_COLLECTION_NAME"), + database_id=os.getenv("APPWRITE_METADATA_DATABASE_ID"), + database_name=os.getenv("APPWRITE_METADATA_DATABASE_NAME"), + ) + + class UNFAOPostProcessorManager(PostprocessorManager, ForecastingModelManager): def __init__( self, @@ -82,7 +179,6 @@ def __init__( logger.info(f"Initializing {self.__class__.__name__}") self._forecast_resolution = None # {target: TargetLease}, set by _read self._historical_frame = None # views_frames.FeatureFrame, set by _read - self.ensemble_path_manager = None def _read_historical_frame(self): """#126: historical actuals as a views_frames.FeatureFrame — the first @@ -118,67 +214,16 @@ def _read_historical_data(self): """ # Declaration first: the check reads the queryset, not the loader, so a # refused config must not pay for loader construction. - launch_config.assert_frame_native_historical( - declared_data_format(self._model_path.get_queryset()) - ) + # + # Read ONCE, and distinguish "could not import it" from "it declares the wrong + # thing" (register C-83). Passing None straight into `declared_data_format` + # turns a failed import into a confident, wrong claim about the config. + queryset = self._model_path.get_queryset() + launch_config.assert_queryset_was_importable(queryset) + launch_config.assert_frame_native_historical(declared_data_format(queryset)) self._initialize_data_loader() self._read_historical_frame() - def _prod_forecasts_datastore(self) -> DatastoreModule: - """The shared internal store (ADR-013's 'shared shelf'), configured from the - launcher-assembled environment (validated fail-loud; þing-01 #134 — no dotenv - is loaded here). - - pipeline-core's ``DatastoreModule.get_predictions_by_metadata`` injects an - automatic ``name == model_name`` filter on every lookup. The contract read - must **not** have it: ADR-013 artifacts are named - ``{run_id}__{target}__m{month}.arrow.parquet`` (never the bare ensemble - name), so an injected ``name == "rusty_bucket"`` matches nothing and also - clobbers the wire layer's own run-id / target / name filters. Suppressed - unconditionally below — the retired legacy reader was the only caller that - needed it on (#149).""" - ensemble_name = self.configs.get("ensemble", None) - if not ensemble_name: - err_msg = "Ensemble name must be provided in configs with the `ensemble` key for forecasting. Cannot proceed." - logger.error(err_msg) - raise ValueError(err_msg) - self.ensemble_path_manager = EnsemblePathManager(ensemble_name_or_path=ensemble_name, validate=False) - # ensemble_configs = EnsembleManager( - # ensemble_path=self.ensemble_path_manager, - # ).configs - - # loa = ensemble_configs.get("level", None) - loa = "pgm" - if not loa: - err_msg = "level must be defined in the ensemble configurations (e.g, pgm, cm). Cannot proceed." - logger.error(err_msg) - raise ValueError(err_msg) - - appwrite_env.assert_env_declared( - appwrite_env.CONNECTION_ENV + appwrite_env.PROD_FORECASTS_ENV, - store="production_forecasts datastore", - ) - appwrite_config = AppwriteConfig( - path_manager=self.ensemble_path_manager, - endpoint=os.getenv("APPWRITE_ENDPOINT"), - project_id=os.getenv("APPWRITE_DATASTORE_PROJECT_ID"), - credentials=os.getenv("APPWRITE_DATASTORE_API_KEY"), - auth_method="api_key", - cache_ttl_hours=24, - bucket_id=os.getenv("APPWRITE_PROD_FORECASTS_BUCKET_ID"), - bucket_name=os.getenv("APPWRITE_PROD_FORECASTS_BUCKET_NAME"), - collection_id=os.getenv("APPWRITE_PROD_FORECASTS_COLLECTION_ID"), - collection_name=os.getenv("APPWRITE_PROD_FORECASTS_COLLECTION_NAME"), - database_id=os.getenv("APPWRITE_METADATA_DATABASE_ID"), - database_name=os.getenv("APPWRITE_METADATA_DATABASE_NAME"), - ) - datastore = DatastoreModule(appwrite_file_manager_config=appwrite_config) - # Suppress the automatic name==model_name filter (see docstring). model_path - # is used by DatastoreModule only for that injection and for uploads; the - # contract read neither uploads nor performs any model-scoped lookup. - datastore.model_path = None - return datastore - def _read_forecast_data_contract(self): """ADR-013 contract inbound (epic #105; streaming since the run-0 OOM fix): RESOLVE the newest fully-manifested run — manifests + pinned shard @@ -187,7 +232,7 @@ def _read_forecast_data_contract(self): released). The `wire/` package owns the policy; this method only adapts the store (DIP) and declares the product facts (region curation + coverage expectations live in the lease, where frames exist).""" - port = _ContractStorePort(self._prod_forecasts_datastore()) + port = _ContractStorePort(_build_prod_forecasts_store(self.configs.get("ensemble"))) region = self.configs.get("region") self._forecast_resolution = source_selection.resolve_run( port, @@ -307,7 +352,7 @@ def _save_contract(self) -> dict: # file itself. lookup = gaul_lookup.load() upload_enabled = bool(self.configs.get("wire_upload_enabled", product.UPLOAD_ENABLED)) - store = _ContractStorePort(self._unfao_datastore()) if upload_enabled else None + store = _ContractStorePort(_build_partner_store(self._model_path)) if upload_enabled else None # The wire is partner-neutral (#153): the manager supplies FAO's product # facts explicitly rather than the mechanism reaching for them. summary = wire_sink.deliver_run( @@ -333,7 +378,15 @@ def _save_contract(self) -> dict: store.upload( hist_path, filename=hist_path.name, - name=self._model_path.model_name, + # The DECLARED consumer name, not `self._model_path.model_name` + # (register C-77). Both resolve to the same string today, because + # `model_name` is the views-models directory name and that directory + # happens to match — but only one of them is a declaration. The other + # is a filesystem coincidence in a different repository, and a + # directory rename there would strand this artifact silently: the + # consumer filters on the declared name, finds nothing, and reports + # an empty endpoint rather than an error (ADR-013 §4.1a). + name=product.CONSUMER_DOCUMENT_NAME, doc_type="model", category="historical", loa="pgm", @@ -349,29 +402,6 @@ def _save_contract(self) -> dict: summary["historical"] = hist_path.name return summary - def _unfao_datastore(self) -> DatastoreModule: - """The FAO-facing store (`unfao_bucket`).""" - return DatastoreModule(appwrite_file_manager_config=self._unfao_appwrite_config()) - - def _unfao_appwrite_config(self) -> AppwriteConfig: - appwrite_env.assert_env_declared( - appwrite_env.CONNECTION_ENV + appwrite_env.UNFAO_ENV, store="unfao_bucket datastore" - ) - return AppwriteConfig( - path_manager=self._model_path, - endpoint=os.getenv("APPWRITE_ENDPOINT"), - project_id=os.getenv("APPWRITE_DATASTORE_PROJECT_ID"), - credentials=os.getenv("APPWRITE_DATASTORE_API_KEY"), - auth_method="api_key", - cache_ttl_hours=24, - bucket_id=os.getenv("APPWRITE_UNFAO_BUCKET_ID"), - bucket_name=os.getenv("APPWRITE_UNFAO_BUCKET_NAME"), - collection_id=os.getenv("APPWRITE_UNFAO_COLLECTION_ID"), - collection_name=os.getenv("APPWRITE_UNFAO_COLLECTION_NAME"), - database_id=os.getenv("APPWRITE_METADATA_DATABASE_ID"), - database_name=os.getenv("APPWRITE_METADATA_DATABASE_NAME"), - ) - def _save(self) -> dict: """Deliver the run — ADR-013 contract only (#149).""" return self._save_contract()