diff --git a/.github/workflows/run_pytest.yml b/.github/workflows/run_pytest.yml index 989dd50..e8dd1b2 100644 --- a/.github/workflows/run_pytest.yml +++ b/.github/workflows/run_pytest.yml @@ -33,37 +33,53 @@ jobs: - name: Install dependencies run: | - # --all-extras is REQUIRED, not tidiness. `views-frames` and `pandas` are - # optional extras, and tests/test_metric_frame.py and - # tests/test_evaluation_report.py both guard with a MODULE-LEVEL - # pytest.importorskip. Without the extras those files are skipped whole and + # --all-extras is REQUIRED, not tidiness. `views-frames` is an optional + # extra, and tests/test_metric_frame.py guards with a MODULE-LEVEL + # pytest.importorskip. Without the extra that file is skipped whole and # CI reports green without ever touching the MetricFrame cross-repo contract # (ADR-022 §1). Guarded by tests/test_falsification_ci_coverage_gap.py. poetry install --all-extras # Nothing else ever exercises "works without the dev group": the local env and the - # steps below always carry scikit-learn. `poetry sync --only main` removes the dev - # group (scikit-learn among it) and keeps the extras; the probe then asserts that - # scikit-learn is genuinely ABSENT — not merely unloaded, which would be vacuous — - # and that importing the package does not load pandas, which the extras install - # (ADR-011; C-05). The reinstall restores the dev group; the verify step below - # proves the extras are present, and the pytest step needs the dev group or it - # cannot collect. + # steps below always carry scikit-learn and pandas. `poetry sync --only main` + # removes the dev group (both among it) and keeps the extras; the probe then + # asserts that scikit-learn and pandas are genuinely ABSENT — not merely unloaded, + # which would be vacuous — and then evaluates a frame and emits a MetricFrame, + # because the emit path imports everything it needs at call time and an + # import-only probe would not execute a line of it (ADR-011; C-05, C-40). The + # "does not LOAD pandas when it is present" half is the purity guard in + # tests/test_metric_calculators.py, which fails rather than skips if pandas is + # missing from the dev group. The reinstall restores the dev group; the verify + # step below proves the extra is present, and the pytest step needs the dev group + # or it cannot collect. - name: Prove the runtime-only install works run: | set -e poetry sync --only main --all-extras - poetry run python -c "import sys, importlib.util as u, views_evaluation, views_evaluation.evaluation.metric_frame; assert u.find_spec('sklearn') is None, 'scikit-learn still installed'; assert 'pandas' not in sys.modules, 'import loaded pandas'" + poetry run python - <<'PY' + import importlib.util as u + import numpy as np + from views_evaluation import EvaluationFrame, NativeEvaluator + assert u.find_spec('sklearn') is None, 'scikit-learn still installed' + assert u.find_spec('pandas') is None, 'pandas still installed' + ids = {k: np.array(v) for k, v in {'time': [1, 1], 'unit': [1, 2], 'origin': [0, 0], 'step': [1, 1]}.items()} + cfg = {'steps': [1], 'classification_targets': ['t'], 'classification_point_metrics': ['AP', 'Brier_cls_point']} + report = NativeEvaluator(cfg).evaluate(EvaluationFrame(np.array([0., 1.]), np.array([[0.2], [0.8]]), ids, metadata={'target': 't'})) + mf = report.to_metric_frame(model_id='ci', run_type='test', timestamp=1758153600, evaluation_timestamp='2026-09-18T00:00:00') + assert isinstance(mf.metadata.provenance.timestamp, int), 'timestamp is int in the cross-repo contract (C-47)' + assert mf.n_rows == len(cfg['classification_point_metrics']) * 2 * 3, mf.n_rows # metrics x (group + mean) x schemas + print('runtime-only install: evaluate + to_metric_frame OK, rows =', mf.n_rows) + PY poetry install --all-extras - # Proves the extras ARRIVED, not merely that --all-extras was requested. + # Proves the extra ARRIVED, not merely that --all-extras was requested. # `poetry install` exits 0 even if it resolves without an optional package, and - # every downstream guard is a module-level pytest.importorskip that turns absence + # the downstream guard is a module-level pytest.importorskip that turns absence # into silence. Without this step a partial install silently shrinks the suite # and CI still reports green. Guarded by # tests/test_falsification_extras_actually_installed.py. - name: Verify optional extras are installed - run: poetry run python -c "import views_frames, pandas" + run: poetry run python -c "import views_frames" - name: Run tests run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index d5d26e7..f362b77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,97 @@ _Nothing yet._ --- +## [2.0.0] — 2026-09-18 + +**MAJOR** (ADR-022 §5): this release removes the surface 1.1.0 deprecated, one release +cycle later as §2 requires. It is the end of the pandas-free epic (#66): `import +views_evaluation` loads neither pandas nor scikit-learn, no public method returns a +pandas object, and the runtime dependencies are numpy, scipy and the optional `frames` +extra. `MetricFrame` (on disk and in memory), `to_dict()` and `get_schema_results()` are +unchanged from 1.1.0: every test on them passes unmodified. Stories S7 (#63) and S8 +(#72); tracking #73. + +### Removed + +- **Removed: `EvaluationReport.to_dataframe()`** and, with it, `BaseEvaluationMetrics.evaluation_dict_to_dataframe()`, + the three `make_*_evaluation_dict` factories (unused since 0.4.0 removed `EvaluationManager`, + their only caller; never documented, never warned on — they go as private cleanup), and + the **`dataframe` extra** (`pip install views-evaluation[dataframe]` is no longer a thing + to install). Breaking. The `DeprecationWarning` for the removed `to_dataframe()` and + `evaluation_dict_to_dataframe()` shipped in 1.1.0 (see that section's *Deprecated* + entry). views-pipeline-core, the one production caller: its `development` branch stopped + calling it (their #513, observed 2026-09-16), but as of 2026-09-18 that is in no release + of theirs — their released 3.2.0 still calls the removed `to_dataframe()`, and their + `^1.0.0` pin excludes this release until they move (their #515, after views-reporting). + **Migration:** + build the table in the caller — + `pd.DataFrame.from_dict(report.to_dict()['schemas'][schema], orient='index')`. Same + values; not byte-identical, as the 1.1.0 entry spells out: the old method dropped any + column that was `nan` in every group (register **C-40**, closed by this removal — + `to_dict()` and `to_metric_frame()` now are the only export paths and agree on the + metric set by construction), ordered columns by dataclass field, and kept an empty + group as a `nan` row. `schema='raw'` callers use `to_dict()['schemas']`. + `get_schema_results()` and the four typed dataclasses' fields are unchanged (the + dataclasses lose the four inherited helpers named above and nothing else). +- `examples/benchmark_probabilistic_scaling.py`, a relic that never imported this package. + +### Changed + +- **Nothing under `views_evaluation/` imports pandas any more**, lazily or otherwise; the + package's runtime dependencies are numpy, scipy and the optional `frames` extra. pandas + moves to the dev group so the import-purity guard, which fails rather than skips when + pandas is absent, keeps something to prove. The CI runtime-only install step now asserts + pandas is *absent* alongside scikit-learn. + +### Release checklist (ADR-022 §7) + +- [x] **Does this release do anything rule 2 governs — remove an `__all__` symbol, remove + a supported config key, narrow an accepted input, or change a raised exception type? + If so, did a `DeprecationWarning` ship at least one release ago?** It removes a + documented method of an `__all__` symbol (the removed `EvaluationReport.to_dataframe()`) + and the helper behind it. Yes: the `DeprecationWarning` naming `to_dict()` as the replacement + shipped in **1.1.0** (published 2026-09-18; see its *Deprecated* entry), one full release + cycle before this one, and both release notes record it (§2 rule 3). Said plainly: §2 + counts cycles in releases, not days, and both releases were cut on the same date, so a + consumer who never installed 1.1.0 never saw the warning at runtime. The release notes + and the pre-tag notifications below are that consumer's notice; the maintainer chose + two same-day releases over one so the warning exists in a published artifact and the + §2 record is honest rather than skipped. The three + `make_*_evaluation_dict` factories were never documented behaviour of any `__all__` + symbol (§1) and had no caller since 0.4.0; they go as private cleanup and are listed + above for completeness, not because rule 2 reached them. No config key is removed, no + accepted input narrowed, no raised exception type changed. +- [x] **Does this release make previously-accepted input fail? If so, are the release + notes explicit, and have known consumers been notified?** Yes — a call to + `to_dataframe()` (any schema, including `'raw'`) now raises `AttributeError`, and + `pip install views-evaluation[dataframe]` no longer selects an extra. The migration is + above. Known consumers, measured 2026-09-18: **views-pipeline-core** — their released + 3.2.0 still calls the removed `report.to_dataframe()` in `managers/evaluation/stage.py`; + their `development` branch does not (their #513) and keeps a deliberate tripwire test + that calls it; their pin `^1.0.0` excludes this release until they move (their #515). + **views-reporting** — `main` and `development` pin `>=1.0.0,<3.0.0`; their released + v0.3.3 still pins `<2.0.0`, so an environment built from that tag excludes this + release too; they call nothing that changed. Both were + notified **before the tag** on 2026-09-18: views-platform/views-pipeline-core#515 + (comment 5733334619) and views-platform/views-reporting#289 (comment 5733334868); + the comments are the committed artifact. +- [x] **Does the version bump match the change class (rule 5)?** `1.1.0` → `2.0.0`, + MAJOR. Removing a documented method is breaking under §1/§5; nothing here is argued + MINOR-eligible. +- [x] **Do the release notes list every breaking change with its migration?** Yes — the + one breaking change is the *Removed* entry, with the `to_dict()` recipe, the three ways + it differs from what the old method returned, and the `schema='raw'` replacement. The + dependency change (pandas to dev-only) removes nothing any consumer imported through + this package: the `dataframe` extra was the only way pandas arrived, and it is the + removed surface. +- [x] **Does `MetricFrame`'s format or axis vocabulary change? If so, has it been agreed + with views-reporting and views-pipeline-core?** No. Axes, `MEAN_GROUP_ID`, + `SCHEMA_TO_EVAL_TYPE`, dtype and the `save`/`load` layout are unchanged; the CI + runtime-only step emits a frame from the release commit with pandas and scikit-learn + removed. + +--- + ## [1.1.0] — 2026-09-18 The first release cut through the gated publish path (S2, #68): `publish_package.yml` now diff --git a/README.md b/README.md index 0ea0726..fea074c 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ The evaluation ontology has been updated to be more explicit and task-specific. The **VIEWS Evaluation** repository provides a standardized framework for **assessing time-series forecasting models** used in the **VIEWS conflict prediction pipeline**. It ensures consistent, robust, and interpretable evaluations through **metrics tailored to conflict-related data**, which often exhibit **right-skewness and zero-inflation**. -The library is built on a **three-layer architecture** with a framework-agnostic NumPy core, ensuring that all mathematical evaluation logic is independent of Pandas or any other data-frame library. +The library is built as a pure NumPy + SciPy core (ADR-011's Level 0, which includes `EvaluationReport`) plus a thin emit layer — `MetricFrame` and `EvaluationReport.to_metric_frame()`, the emit path of the logging standard §5.1 — with orchestration left to the caller (ADR-011's Level 2, external). All mathematical evaluation logic is independent of Pandas or any other data-frame library — nothing in the package imports one. --- @@ -119,8 +119,8 @@ report = evaluator.evaluate(ef) # 3. Access results report.to_dict() # {'target', 'task', 'pred_type', 'schemas': {'step': {...}, ...}} report.get_schema_results("month") # typed metrics dataclass -# to_dataframe() is deprecated and removed in 2.0.0; build a DataFrame from -# to_dict()['schemas'][schema] in the caller if you need one. +# Need a DataFrame? Build it in the caller: +# pd.DataFrame.from_dict(report.to_dict()['schemas']['month'], orient='index') ``` > For the full walkthrough including input formatting and sample evaluation, see [`documentation/integration_guide.md`](documentation/integration_guide.md). @@ -257,7 +257,7 @@ pip install views_evaluation --- ## 🏗 **Architecture** -The library follows a strict three-layer architecture (ADR-011): +The library follows the strict layering of ADR-011; the first two levels live in this repository: ``` Level 0 — Pure Core (NumPy + SciPy; no dataframe libraries) @@ -268,7 +268,7 @@ Level 0 — Pure Core (NumPy + SciPy; no dataframe libraries) Profiles Named hyperparameter sets (base, hydranet_ucdp, ...) Level 1 — Bridge / Emit - EvaluationReport Results container with dict / MetricFrame export (DataFrame export deprecated) + EvaluationReport Results container with dict / MetricFrame export MetricFrame Typed, provenance-stamped evaluation-of-record (views-frames ADR-020) Level 2 — Orchestration diff --git a/documentation/ADRs/041_evaluation_output_schema.md b/documentation/ADRs/041_evaluation_output_schema.md index 4c85f50..0526b5c 100644 --- a/documentation/ADRs/041_evaluation_output_schema.md +++ b/documentation/ADRs/041_evaluation_output_schema.md @@ -22,7 +22,7 @@ The 2025 version of this ADR asserted that *"views-evaluation returns a structur - `to_dict()` — nested plain dict; the pandas-free, extra-free path. - `get_schema_results(schema)` — mapped onto the typed metrics dataclasses. -- `to_dataframe(schema)` — pandas DataFrame; requires the optional `dataframe` extra. **Deprecated; removed in 2.0.0** (ADR-022 §2): build the DataFrame from `to_dict()` in the caller. +- `to_dataframe(schema)` — *removed in 2.0.0* after a 1.1.0 `DeprecationWarning` (ADR-022 §2). It returned a pandas DataFrame behind the optional `dataframe` extra; a caller builds one from `to_dict()` now. `views-evaluation` does **not** serialise these to JSON or render HTML. That remains the orchestrator's job, and the original rationale still holds for them: formatting and run context belong where the run is orchestrated. diff --git a/documentation/CICs/EvaluationReport.md b/documentation/CICs/EvaluationReport.md index bc15c82..6894652 100644 --- a/documentation/CICs/EvaluationReport.md +++ b/documentation/CICs/EvaluationReport.md @@ -2,14 +2,14 @@ **Status:** Active **Owner:** Evaluation Core -**Last reviewed:** 2026-08-02 +**Last reviewed:** 2026-09-18 (2.0.0: DataFrame surface removed) **Related ADRs:** ADR-010 (Ontology), ADR-041 (Output Schema), views-frames ADR-020 (MetricFrame contract home) --- ## 1. Purpose -A structured, framework-agnostic container for evaluation results. It decouples the mathematical outcomes from their final presentation format (DataFrames, JSON, etc.). +A structured, framework-agnostic container for evaluation results. It decouples the mathematical outcomes from their final presentation format (nested dictionaries for JSON, typed dataclasses, and the `MetricFrame` evaluation-of-record; no DataFrame since 2.0.0). --- @@ -17,7 +17,7 @@ A structured, framework-agnostic container for evaluation results. It decouples - This class does **not** calculate metrics. - This class does **not** handle data alignment. -- This class does **not** depend on Pandas internally (though it may provide conversion methods). +- This class does **not** depend on pandas, and since 2.0.0 offers no conversion to it. --- @@ -25,8 +25,8 @@ A structured, framework-agnostic container for evaluation results. It decouples - **Multi-Schema Storage**: Guarantees storage of results across Month, Sequence, and Step schemas. - **Traceability**: Maintains metadata about the target, task type, and prediction type. -- **Representation Agnosticism**: Provides a standard internal representation that can be converted to external formats (Dictionary or Pandas). -- **Field Validation**: Guarantees that computed metric names match dataclass fields. Raises `ValueError` with an actionable message if a metric is computed but has no corresponding field in the typed container (FM1 guard). +- **Representation Agnosticism**: Provides a standard internal representation that can be converted to external formats (dictionary or `MetricFrame`). +- **Field Validation** (`get_schema_results()` only): guarantees that computed metric names match dataclass fields on the typed path. Raises `ValueError` with an actionable message if a metric is computed but has no corresponding field in the typed container (FM1 guard). `to_dict()` and `to_metric_frame()` carry every metric the results hold unvalidated; for catalog metrics the membership check in `NativeEvaluator` and `test_metric_membership_matches_dataclass_fields` close the gap. --- @@ -46,7 +46,6 @@ A structured, framework-agnostic container for evaluation results. It decouples - **`to_dict()`**: Returns a nested dictionary `{target, task, pred_type, schemas: {...}}` for persistence (ADR-041). - **`get_schema_results(schema)`**: Returns a dict mapping group keys to typed metrics dataclass instances. -- **`to_dataframe(schema)`**: Returns a Pandas DataFrame. **Deprecated; removed in 2.0.0** (ADR-022 §2): every call emits a `DeprecationWarning` naming `to_dict()` as the replacement. Without the `dataframe` extra it raises `ModuleNotFoundError` naming `pip install views-evaluation[dataframe]` (C-44); no log — this path computes in memory. `schema='raw'` returns the internal dict (same object as `to_dict()['schemas']`). - **`to_metric_frame(...)`**: Returns a `MetricFrame` (the typed, transportable, provenance-stamped evaluation-of-record per views-frames ADR-020). Flattens the nested results to rows keyed by `(eval_type, target, metric, group_id, partition, level)`, emitting per-group rows plus a `group_id="mean"` cross-group aggregate. All identity (`model_id`/`run_id`/`data_version`/`partition`/`level`) is **injected** by the caller; `scoring_code_version` (defaults to `default_scoring_code_version()`: the installed version, plus `+g` when running from this repository's own checkout — see CICs/MetricFrame.md) and `evaluation_timestamp` are stamped into the MetricFrame's own metadata (the generic header stays in the reused `views_frames.FrameMetadata`). Requires the optional `views-frames` dependency. Purely additive — `to_dict()` is unaffected. --- @@ -54,10 +53,9 @@ A structured, framework-agnostic container for evaluation results. It decouples ## 6. Failure Modes and Loudness - Raises `KeyError` if a requested schema is not found in the report. -- Raises `ValueError` if a computed metric name has no corresponding field in the typed metrics dataclass (FM1 guard against silent metric loss). +- `get_schema_results()` raises `ValueError` if a computed metric name has no corresponding field in the typed metrics dataclass (FM1 guard against silent metric loss); nothing is logged. - Fails loud if the input result structure is inconsistent. - `to_metric_frame()` raises `ImportError` (loud, actionable) if the optional `views-frames` dependency is not installed. -- `to_dataframe()` raises `ModuleNotFoundError` naming the `dataframe` extra if pandas is not installed (C-44). Deprecated path; gone in 2.0.0. - `to_metric_frame()` raises `ValueError` if the report contains **no metric values for any schema** (ADR-015 ruling 6, risk register C-30). Such an emit previously produced a structurally valid **zero-row** MetricFrame that satisfied `assert_frame_envelope` and persisted to disk as a legitimate-looking audit artifact recording nothing. The message names the target and which schemas were present-but-empty. A **partial** report — at least one metric value in any schema — still emits normally. **Loudness note:** this class is Level 0 and maintains no logger, with **one deliberate exception**: the `to_metric_frame()` emit path logs at `ERROR` before raising, because it belongs to Level 1 (it produces the persisted, cross-repo evaluation-of-record). Logging follows the emit path, not the file (logging standard §5.1). No other raise in this class logs. @@ -75,7 +73,6 @@ A structured, framework-agnostic container for evaluation results. It decouples ```python report = EvaluationReport(target="ged_sb_best", task="regression", pred_type="point", results=native_results) -df = report.to_dataframe(schema="month") # pd.DataFrame — deprecated, removed in 2.0.0 data = report.to_dict() # nested dict schema = report.get_schema_results("month") # dict → typed metrics dataclass ``` @@ -84,7 +81,7 @@ schema = report.get_schema_results("month") # dict → typed metrics datacla ## 9. Examples of Incorrect Usage -- Calling `to_dataframe(...)` at all in new code — it is deprecated and goes in 2.0.0; build the DataFrame from `to_dict()['schemas'][schema]`. (`schema='raw'` additionally returns the internal dict, not a DataFrame.) +- Expecting a DataFrame from the report — no method returns one (`to_dataframe()` was removed in 2.0.0); build it in the caller from `to_dict()['schemas'][schema]`. - Adding a new metric to `METRIC_CATALOG` without adding a corresponding field to the typed metrics dataclass — the FM1 guard will raise `ValueError`. - Treating the report as mutable and modifying `_results` after construction. @@ -92,23 +89,22 @@ schema = report.get_schema_results("month") # dict → typed metrics datacla ## 10. Test Alignment -- **Green:** `tests/test_evaluation_report.py` — construction, schema access, to_dict, to_dataframe (each call asserted to warn). `tests/test_metric_frame.py` — `to_metric_frame()` for all 4 cells, envelope conformance, save/load round-trip, provenance split. -- **Beige:** `tests/test_evaluation_report.py` — empty schemas, single-entry schemas, the `to_dataframe()` DeprecationWarning (one per call, attributed to the caller). `tests/test_metric_frame.py` — empty/fully-empty schema, NaN values, run_id None at emit. -- **Red:** `tests/test_evaluation_report.py` — missing schema keys, field mismatch (FM1 guard). `tests/test_adversarial_inputs.py::TestOptionalExtraAbsentRed` — `to_dataframe()` with pandas absent names the extra (C-44). `tests/test_metric_frame.py` — fail-loud MetricFrame construction. +- **Green:** `tests/test_evaluation_report.py` — construction, schema access, to_dict. `tests/test_metric_frame.py` — `to_metric_frame()` for all 4 cells, envelope conformance, save/load round-trip, provenance split. +- **Beige:** `tests/test_evaluation_report.py` — empty schemas, single-entry schemas. `tests/test_metric_frame.py` — empty/fully-empty schema, NaN values, run_id None at emit. +- **Red:** `tests/test_evaluation_report.py` — missing schema keys, field mismatch (FM1 guard, asserted silent), the public surface pinned as an allowlist (nothing removed in 2.0.0 comes back under any name). `tests/test_adversarial_inputs.py::TestOptionalExtraAbsentRed` — `to_metric_frame()` with views-frames absent raises `ImportError` naming the `frames` extra and logs it on the emit-path logger. `tests/test_metric_frame.py` — fail-loud MetricFrame construction. --- ## 11. Evolution Notes -- The `to_dataframe()` method imports Pandas lazily. Deprecated with a `DeprecationWarning` (ADR-022 §2); removed in 2.0.0 together with `BaseEvaluationMetrics.evaluation_dict_to_dataframe()` and the `dataframe` extra (epic #66, story #63). The `get_schema_results()` dataclass path stays. +- `to_dataframe()` (a lazily-imported pandas DataFrame; `schema='raw'` returned the internal dict) was deprecated in 1.1.0 with a `DeprecationWarning` and removed in 2.0.0 together with `BaseEvaluationMetrics.evaluation_dict_to_dataframe()`, the three unused `make_*_evaluation_dict` factories and the `dataframe` extra (ADR-022 §2; epic #66, story #63; register C-40, C-44). The `get_schema_results()` dataclass path stays. - The `_metrics_map` mapping 4 (task, pred_type) combinations to dataclass types is stable but must be extended if new task types are added. --- ## 12. Known Deviations -- **Lazy Pandas import:** `to_dataframe()` imports `pandas` at call time (gated on `importlib.util.find_spec`, as `to_metric_frame()` is), which means the Level 1 bridge concern leaks into what is otherwise a Level 0 component. Tolerated for one deprecation cycle; gone in 2.0.0. -- **Lazy views-frames import:** `to_metric_frame()` imports `views_frames` at call time (gated on `importlib.util.find_spec`), the same Level-1 bridge compromise as the Pandas case. The core stays installable without the `frames` extra (ADR-011 minimal core); the method fails loud if the extra is absent. +- **Lazy views-frames import:** `to_metric_frame()` imports `views_frames` at call time (gated on `importlib.util.find_spec`), a Level-1 bridge concern inside an otherwise Level-0 component — the one such bridge left after 2.0.0 removed the pandas one. The core stays installable without the `frames` extra (ADR-011 minimal core); the method fails loud if the extra is absent. - **Legacy dataclass coupling:** `get_schema_results()` wraps results in legacy dataclass instances (`RegressionPointEvaluationMetrics`, etc.) from `metrics.py`. If a metric is computed but has no field in the dataclass, the FM1 guard raises. This means new metrics require coordinated updates to both `metric_catalog.py` and `metrics.py`. --- diff --git a/documentation/CICs/NativeEvaluator.md b/documentation/CICs/NativeEvaluator.md index 75e8c25..63086e5 100644 --- a/documentation/CICs/NativeEvaluator.md +++ b/documentation/CICs/NativeEvaluator.md @@ -96,7 +96,6 @@ report = evaluator.evaluate(ef) # legacy_compatibility defaults to # Access results step_dict = report.to_dict()['schemas']['step'] # raw nested dict -# to_dataframe('month') still works but is deprecated (removed in 2.0.0) schema = report.get_schema_results('time_series') # dict → typed metrics dataclass ``` diff --git a/documentation/integration_guide.md b/documentation/integration_guide.md index 107f9d1..6187afa 100644 --- a/documentation/integration_guide.md +++ b/documentation/integration_guide.md @@ -161,9 +161,6 @@ report.pred_type # str: 'point' or 'sample' report.to_dict() # {'target': ..., 'task': ..., 'pred_type': ..., # 'schemas': {'month': {...}, 'time_series': {...}, 'step': {...}}} -report.to_dataframe(schema) # DEPRECATED, removed in 2.0.0 — use - # to_dict()['schemas'][schema]; emits DeprecationWarning - report.get_schema_results('month') # dict mapping key → typed metrics dataclass ``` diff --git a/documentation/standards/logging_and_observability_standard.md b/documentation/standards/logging_and_observability_standard.md index 203ef33..13ef2e2 100644 --- a/documentation/standards/logging_and_observability_standard.md +++ b/documentation/standards/logging_and_observability_standard.md @@ -128,7 +128,7 @@ The following must be logged: > **Scope note (revised 2026-08-02):** logging responsibility in this repository splits by architectural level (ADR-011). The split is stated positively below so it cannot be re-derived by guesswork as new components are added — the previous wording named three classes and predated `MetricFrame`, which left Level 1 uncovered by omission rather than by decision. > > **Level 0 — exempt. Must NOT maintain loggers.** -> `evaluation_frame.py`, `native_evaluator.py`, `evaluation_report.py`, `metric_catalog.py`, `native_metric_calculators.py`. +> `evaluation_frame.py`, `native_evaluator.py`, `evaluation_report.py`, `metric_catalog.py`, `native_metric_calculators.py`, `metrics.py`, `config_schema.py` (the last two added 2026-09-18 so this list, the logging guard and the import-purity guard classify the same files). > These are pure math and pure registry. They compute and validate; they do not act on the world. They rely on exception propagation per ADR-013, and logging responsibility sits at the orchestration layer (e.g. `views-pipeline-core`). Adding a logger here is a violation, not an improvement: it would duplicate what the orchestrator already records and put I/O in the numeric core. > > **Level 1 — must log at `ERROR` before raising.** diff --git a/documentation/standards/physical_architecture_standard.md b/documentation/standards/physical_architecture_standard.md index 6657855..1a7bf80 100644 --- a/documentation/standards/physical_architecture_standard.md +++ b/documentation/standards/physical_architecture_standard.md @@ -113,7 +113,7 @@ This is a **future evolution path**, not a current mandate. The current bundling - **Explicit imports:** Avoid `from module import *`. - **Circular dependency guard:** Follow ADR-011 layering. Level 0 modules must not import from Level 1 or Level 2. -- **Lazy imports for Pandas:** Pandas is imported inside methods (e.g. the deprecated `to_dataframe()`, gone in 2.0.0) rather than at module level in Level 0/1 code. +- **No pandas anywhere in the package:** since 2.0.0 nothing under `views_evaluation/` imports pandas, lazily or otherwise (the last two sites, `EvaluationReport.to_dataframe()` and `BaseEvaluationMetrics.evaluation_dict_to_dataframe()`, were removed after the deprecation cycle; `tests/test_metric_calculators.py` polices the whole package by AST). The one lazy **Level-1 bridge** that remains is `views_frames` inside `to_metric_frame()`, gated on `importlib.util.find_spec`; other function-level imports (numpy, logging, this package's own `metric_frame`) are ordinary call-time imports on that same path, not bridges to a framework. --- diff --git a/examples/benchmark_probabilistic_scaling.py b/examples/benchmark_probabilistic_scaling.py deleted file mode 100644 index 2bbebd2..0000000 --- a/examples/benchmark_probabilistic_scaling.py +++ /dev/null @@ -1,69 +0,0 @@ -import time -import numpy as np -import pandas as pd -import properscoring as ps -import psutil -import os -from sklearn.metrics import mean_squared_error - -def get_memory_usage(): - process = psutil.Process(os.getpid()) - return process.memory_info().rss / (1024 * 1024) - -def benchmark_scaling(): - n_rows = 200000 - n_samples = 100 - - print(f"Benchmark Configuration: N_rows={n_rows}, N_samples={n_samples}") - - y_true = np.random.rand(n_rows) - y_pred = np.random.rand(n_rows, n_samples) - - print("") - print("--- CRPS Benchmark ---") - - # Legacy CRPS - df_pred = pd.DataFrame({'pred': [list(row) for row in y_pred]}) - df_actual = pd.DataFrame({'true': [[v] for v in y_true]}) - - start = time.time() - np.mean([ - ps.crps_ensemble(actual[0], np.array(pred)) - for actual, pred in zip(df_actual['true'], df_pred['pred']) - ]) - legacy_duration = time.time() - start - print(f"Legacy CRPS Time: {legacy_duration:.4f}s") - - # Native CRPS - start = time.time() - np.mean(ps.crps_ensemble(y_true, y_pred, axis=1)) - native_duration = time.time() - start - print(f"Native CRPS Time: {native_duration:.4f}s") - print(f"CRPS Speedup: {legacy_duration / native_duration:.1f}x") - - print("") - print("--- MSE Benchmark (The 'Expansion' Pattern) ---") - - # Legacy MSE - start = time.time() - # This mimics calculate_mse in metric_calculators.py - actual_values = np.concatenate(df_actual['true'].values) - pred_values = np.concatenate(df_pred['pred'].values) - actual_expanded = np.repeat(actual_values, [len(x) for x in df_pred['pred']]) - legacy_mse = mean_squared_error(actual_expanded, pred_values) - legacy_duration = time.time() - start - print(f"Legacy MSE Time: {legacy_duration:.4f}s") - - # Native MSE (Vectorized Broadcasting) - start = time.time() - # (N, 1) - (N, S) -> (N, S)**2 -> mean - native_mse = np.mean((y_true[:, np.newaxis] - y_pred)**2) - native_duration = time.time() - start - print(f"Native MSE Time: {native_duration:.4f}s") - print(f"MSE Speedup: {legacy_duration / native_duration:.1f}x") - - print("") - print(f"MSE Parity: {abs(legacy_mse - native_mse) < 1e-10}") - -if __name__ == "__main__": - benchmark_scaling() diff --git a/pyproject.toml b/pyproject.toml index 3285c51..6b497f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "views_evaluation" -version = "1.1.0" +version = "2.0.0" description = "" authors = [ "Xiaolong Sun ", @@ -13,11 +13,9 @@ readme = "README.md" python = ">=3.11,<3.15" numpy = "^1.26.4" scipy = "^1.11" # Level-0 kernels (EMD, Pearson); ADR-011 permits numpy and scipy -pandas = {version = "^1.5.3", optional = true} views-frames = {version = ">=1.10.2,<2", optional = true} [tool.poetry.extras] -dataframe = ["pandas"] frames = ["views-frames"] [tool.poetry.group.dev.dependencies] @@ -25,6 +23,7 @@ pytest = "^9.0.3" ruff = "*" properscoring = "^0.1.0" # Dev-only: used as test oracle for CRPS parity verification scikit-learn = "^1.6.0" # Dev-only: test oracle for AP/MTD parity verification (left runtime 2026-09-18, C-05) +pandas = ">=2.1.1,<3" # Dev-only: the import-purity guard needs pandas *present* to prove it is not loaded (left runtime 2.0.0, C-40). Floor = first cp312 wheels; 1.5.x cannot install on 3.12 [build-system] requires = ["poetry-core"] diff --git a/reports/technical_risk_register.md b/reports/technical_risk_register.md index c1994ef..4f15561 100644 --- a/reports/technical_risk_register.md +++ b/reports/technical_risk_register.md @@ -1,7 +1,7 @@ # Technical Risk Register — views-evaluation **Last updated:** 2026-09-18 -**Total open concerns:** 15 +**Total open concerns:** 14 **Governing ADR:** ADR-023 **Citation convention:** `Location` fields name files and symbols (functions, classes, sections), not line numbers — line numbers drift as soon as anything is inserted above them. @@ -14,6 +14,54 @@ > review then found four gaps in the rewrite (linked-worktree `commondir`, locale-dependent > reads, unchecked ref contents, an unpinned name gate) — all fixed and tested before merge. 18 → 17. +> **2026-09-18 — release 2.0.0 (S8, #72).** The MAJOR that ends the pandas-free epic: `to_dataframe()` +> and the `dataframe` extra gone one release cycle after 1.1.0's `DeprecationWarning` (ADR-022 §2). +> §3.2 notifications posted before the tag and cited by comment ID in the checklist (views-pipeline-core +> #515, views-reporting #289), the second release to do so. Consumer state at the cut, measured: +> views-pipeline-core's released 3.2.0 still calls the removed method and pins `^1.0.0`, so 2.0.0 +> reaches them only when S10 raises the pin; views-reporting's `main` span admits it (their released +> v0.3.3 still pins `<2.0.0`) and calls nothing removed. The release review found fifteen more guard +> defects, all fixed on the release branch and each re-verified red: the S7 import guard keyed its +> allowlist by file stem (so `profiles/__init__.py` inherited the root package's allowance) and read +> a string literal under ANY call (a `nan` package on sys.path would have turned `float("nan")` into a +> red build); both guards are now one walker with one path-keyed allowlist set to the measured census, +> reading literals only under import machinery and its aliases, with `exec`/`eval` forbidden everywhere +> and the runtime probe also exercising `save`/`load`. The doc guard now scans the CHANGELOG section of +> the release being cut (it froze it as history), uses word-anchored removal framings, and lost the +> five block-wide 2.0-era markers that had widened the older guards' exemption. `to_dict()`'s live-dict +> identity, `get_schema_results()` on all three schemas, the report's public surface as an allowlist, +> FM1's silence and the CI probe's `timestamp` type (an ISO string had been passed into the contract's +> `int` field) are now pinned. The independent guard audit then ran 64 mutations and found five weak +> guards; all closed and re-verified red: the runtime probe now evaluates EVERY implemented kernel in +> all four cells (a spec-loaded or `getattr`-imported pandas in an uncalled kernel was invisible), the +> Level-0 crossing check sees `from views_evaluation.evaluation import metric_frame`, the public-surface +> guard probes the removed names behaviourally and bans `__getattr__` (a shim with a split token beat +> the `dir()` listing), the logging guard checks for Logger objects rather than a text pattern, the +> dependency guard handles both pyproject layouts, and the doc guard scans fenced code in the section +> being cut with real line numbers. Residuals stated in the docstrings. Count unchanged. + +> **2026-09-18 — story S7 (#63) closed C-40 and Cluster A's second residue.** `to_dataframe()`, its +> DataFrame helper and the three dead factories are gone; the `dataframe` extra is deleted and +> pandas is a dev-only dependency (the purity guard needs it present). C-15's closed-row rationale +> corrected: the dataclass consumer is `get_schema_results()`, not `to_dataframe()`. The story's +> max review found fifteen defects, all fixed in the same PR rather than registered — among them: +> the per-module purity guard covered five of fourteen modules (a pandas import in the very file +> purged of it passed; now a package-wide AST sweep plus a pin on the exact runtime dependency +> set), nothing tested `to_metric_frame()`'s views-frames-absent branch once the pandas twin was +> deleted (transplanted), no doc guard knew the removed methods (registered, with 2.0-era history +> markers and dated CHANGELOG sections treated as frozen), and the CI runtime-only probe only +> imported (it now evaluates and emits). Every new or widened guard was seen red on an injected +> defect. The independent guard audit then ran 66 mutations and found 16 survivors across four guards +> (aliased or string-literal dynamic imports, a denylist that let `xarray` in, an extras check that +> read names under one TOML layout, a logger name unasserted, and a doc guard whose block-wide +> history exemption and paren-anchored token missed four spellings and two locations); all closed +> — the package-wide guard is now an allowlist per module over static AND string-literal dynamic +> imports, the doc guard matches whole-word tokens with a same-line removal framing and scans CICs +> and `examples/` — and each re-verified red. One residual is stated in the doc guard's docstring +> (a single line that both frames a removal and advertises the method). Consumer state, measured 2026-09-18: views-pipeline-core's released 3.2.0 still calls +> `to_dataframe()`; only their `development` has stopped; their `^1.0.0` pin excludes 2.0.0 until +> S10 (#515). Ships as 2.0.0 (S8). 15 → 14. + > **2026-09-18 — release 1.1.0 (S6, #71).** The first release through the gated publish path > (C-36 mechanical half) and the one that starts ADR-022's clock for `to_dataframe()`. Carries > S1–S5: C-39, C-44, C-05 closed; D-01 (AP `nan`) ships. The release review found and the @@ -29,7 +77,11 @@ > Gaussian — both closed and re-verified red (C-37). First release whose §3.2 > notification was posted **before** the tag and cited by comment ID in the checklist (C-36 > trigger (a) exercised as written); ADR-022 §3 gained a dated clarification on what -> "previously-accepted input" means (D-02). Count unchanged. +> "previously-accepted input" means (D-02). Published 15:30 UTC: publish run 35362812153 ran +> the suite (982 passed, 1 skipped) before the upload — the gate worked on its first release. +> The skip is `test_falsification_session_shutdown.py`'s sibling-checkout guard, which CI has +> never been able to run (no views-pipeline-core checkout beside this repo); pre-existing, a C-37 +> shape, noted here rather than registered. Count unchanged. > **2026-09-18 — story S5 (#64) closed C-05 and Cluster C.** `AP` and `MTD` dispatch to the numpy > kernels; scikit-learn is a dev-only oracle; a guard asserts `import views_evaluation` loads @@ -94,7 +146,7 @@ Root-cause groupings (added 2026-06-26; expanded then largely closed 2026-08-02 by epic #26). Closed clusters are retained as a record of what the root cause was and how it was resolved. -- **Cluster A — No doctrine for degenerate or empty results (Fail-Loud violations)** → **RESOLVED 2026-08-02**, two residues: **C-22** (accepted sentinel) and **C-40** (opened 2026-09-16 — `to_dataframe()` erases the sentinel that ADR-015 contracted; a leak in the doctrine's application, not in the doctrine). **ADR-015** now defines what a computation does when it cannot produce a result and rules on every non-raising path individually, on a **fault-vs-data-property** test. C-02, C-28(a), C-29, C-30, C-32 closed; C-20's truncation half closed. **C-22 remains open as an accepted, documented sentinel rather than a defect** — Pearson's `nan` was ruled a raise and reversed the same day when the raise proved to abort any evaluation of a constant baseline. The cluster's root cause is gone; what is left is a contract, not a silence. +- **Cluster A — No doctrine for degenerate or empty results (Fail-Loud violations)** → **RESOLVED 2026-08-02**, one residue: **C-22** (accepted sentinel). (**C-40**, opened 2026-09-16 — `to_dataframe()` erased the sentinel that ADR-015 contracted — closed 2026-09-18 with the method's removal, S7 #63.) **ADR-015** now defines what a computation does when it cannot produce a result and rules on every non-raising path individually, on a **fault-vs-data-property** test. C-02, C-28(a), C-29, C-30, C-32 closed; C-20's truncation half closed. **C-22 remains open as an accepted, documented sentinel rather than a defect** — Pearson's `nan` was ruled a raise and reversed the same day when the raise proved to abort any evaluation of a constant baseline. The cluster's root cause is gone; what is left is a contract, not a silence. - **Cluster B — Phase-3 deletion left config validation and its documented contracts unowned** → **CLOSED 2026-08-02.** `NativeEvaluator._validate_config` (#31) restored ownership of config validation, deriving the valid key set from `EvaluationConfig` so the schema has one authority; the README was corrected (#38) and a documentation-contract test now guards it (#40). Former members C-02, C-29 closed (+ previously demoted C-21, C-23). - **Cluster C — scipy/sklearn in the Level-0 core** → **CLOSED 2026-09-18.** The declaration half closed 2026-08-02 (#29, C-19); the purity half closed with #64 (C-05): `AP` and `MTD` are numpy transcriptions held to parity against scikit-learn as a dev-only oracle, and `import views_evaluation` no longer loads pandas. scipy stays, as ADR-011 permits. - **Cluster D — MetricFrame evaluation-of-record integrity** → **C-24, C-26** (C-25 closed by #57; C-39 — #57's fix mis-stamping under a consumer's in-repo venv — opened 2026-09-16 and **closed 2026-09-17** by #67). The vacuous-emit exposure (C-30) closed on 2026-08-02 (#34). C-24 and C-26 were deliberately deferred until **after** the first stable release ships (0.5.0, then 1.0.0), so the drift guards pin the contract that actually shipped rather than a moving one. Consumers (views-reporting, pipeline-core) are actively building against this surface. @@ -240,16 +292,6 @@ Root-cause groupings (added 2026-06-26; expanded then largely closed 2026-08-02 --- -### C-40 — `to_dataframe()` silently erases any metric that is `nan` in every group -- **Tier:** 3 (Medium) — one of three export paths disagrees with the other two on which metrics exist; affects every consumer of the DataFrame path and every comparison between paths. Not Tier 2: the DataFrame is a convenience export, not the record; the record (`MetricFrame`) is correct. -- **Description:** `BaseEvaluationMetrics.evaluation_dict_to_dataframe` ends with `df.loc[:, df.notna().any()]`, dropping every column that is `nan` in every row. That predates ADR-015, which now rules `nan` a **documented sentinel** for Pearson (ruling 2) and MCR. **Probed 2026-09-16:** a constant-truth frame scored with `['MSE', 'Pearson']` gives `to_dict()` month results with a finite `MSE` and `Pearson: nan` in *both* month groups, a MetricFrame with both `MSE` and `Pearson` rows (16 rows), and a DataFrame with columns `['MSE']` only. The sentinel and the metric's *name* both vanish, with no warning. A researcher comparing a DataFrame to a MetricFrame, or reading a DataFrame to check which metrics ran, sees a different metric set from the same report. A metric that is `nan` in *some* groups survives, so the erasure is conditional on the data, which is the worst kind of silence: the same config yields different column sets on different inputs. -- **Trigger:** When someone reads `to_dataframe(schema)` to determine which metrics were computed, or joins it against `to_dict()` / a MetricFrame, for any report where a metric is `nan` in every group of that schema — a constant-baseline model scored with Pearson, an all-zero-truth group scored with MCR, or (since 2026-09-18, ADR-015 R9) a classification target with no positive in any group scored with AP. Check whether the metric column exists. -- **Location:** `views_evaluation/evaluation/metrics.py` (`BaseEvaluationMetrics.evaluation_dict_to_dataframe`, the `df.loc[:, df.notna().any()]` filter); `views_evaluation/evaluation/evaluation_report.py` (`to_dataframe`, the caller) -- **Source:** repo-assimilation (2026-09-16), empirically verified -- **Mitigation path:** Remove the column filter so the DataFrame carries the same metric set as the other two paths, with `nan` in place; or, if the filter is wanted, make it opt-in and document that the DataFrame is not the record. Add a test asserting the three export paths agree on the set of metric names for a report containing an all-`nan` metric — no test does today (`tests/test_evaluation_report.py` never constructs a `nan` value). -- **Note:** Residue of causal cluster A alongside C-22: C-22 tracks that the sentinel *exists* and may be misread by consumers; this tracks that one of this library's own exports *erases* it. ADR-015's contract that `nan` means "not computable for this group" is only honoured on two of three paths. **Tracked as GitHub issue #63** (2026-09-16): dies with `to_dataframe()` in the 2.0.0 removal. Its blocker, views-pipeline-core#512, closed the same day via their PR #513 (`9644724`): the caller is gone, verified on their `development` branch. The `DeprecationWarning` was added by S3 (#69); ADR-022's clock starts when the deprecating release is published (S6). - ---- ### C-41 — CI resolves dependencies fresh each run, tests one interpreter of four declared, under a numpy 1.x ceiling - **Tier:** 3 (Medium) — reproducibility and coverage gap that raises the cost of every dependency change and every green-build claim; no correctness impact today. @@ -293,8 +335,8 @@ Maintainer decisions taken on a register concern, recorded here so the reasoning - **Concern:** C-05 — the switch from the scikit-learn AP kernel to the numpy one (epic #66, story #64) is where scikit-learn's inherited `0.0`-plus-`UserWarning` convention would otherwise have become a first-party non-raising return with no ADR-015 ruling; found by `/code-review max` of story #70. - **Options weighed:** (A) `nan`, warning suppressed — chosen; (B) raise — rejected on the R2 evidence (aborts every metric and schema for a routine data condition; tried on Pearson and reversed the same day); (C) keep `0.0`, warning suppressed — rejected because it codifies a wrong number into the evaluation-of-record and silently pulls the `mean` row toward zero by an amount set by the data, not the model (the C-02 shape). - **Consequence (ADR-022 §3):** a behaviour change for any evaluation whose data contains an empty classification group — `nan` for the group, `mean` row over the rest, where before it was `0.0` and a lower mean. Affected at the time of decision: `views-models/ensembles/rusty_bucket` (the one live config requesting `AP`) and views-reporting's canonical classification cell. Both to be named in the release notes of the release that carries the switch (S6, 1.1.0). -- **Ships with:** #64. Closes when 1.1.0 is published with the change in its notes. -- **Residual, now live (found by the code review of #64, 2026-09-18):** C-22's — and the direct `to_dict()` reader it warned about exists: views-pipeline-core averages group values for its WandB scalars with a mean that skips `None` but not `nan` (`views_pipeline_core/modules/wandb/utils.py`, observed on their `development` branch). One empty group makes their `AP_mean` scalar `nan`. The fix is theirs (`nanmean`), named in the release notes, and C-22's trigger is widened to cover it. Also: `to_dataframe()` drops an all-`nan` column (C-40), which now applies to `AP`; the method is deprecated and goes in 2.0.0. +- **Ships with:** #64. **Shipped:** 1.1.0 published to PyPI 2026-09-18 with the change in its notes; this decision's consequence is live. +- **Residual, now live (found by the code review of #64, 2026-09-18):** C-22's — and the direct `to_dict()` reader it warned about exists: views-pipeline-core averages group values for its WandB scalars with a mean that skips `None` but not `nan` (`views_pipeline_core/modules/wandb/utils.py`, observed on their `development` branch). One empty group makes their `AP_mean` scalar `nan`. The fix is theirs (`nanmean`), named in the release notes, and C-22's trigger is widened to cover it. Also, until 2.0.0: `to_dataframe()` dropped an all-`nan` column (C-40), which applied to `AP` too; the method was deprecated in 1.1.0 and removed in 2.0.0 (S7, #63; C-40 closed 2026-09-18). - **Versioning:** filed MINOR on the ADR-022 §1 argument that the `0.0` was never documented behaviour of this library; recorded in ADR-015 R9 so the S6 checklist can answer rule 5 against it. --- @@ -333,7 +375,7 @@ Moved out of the register (no correctness/reliability dimension) — tracked in | C-11 | 3 | pyproject.toml version not bumped for breaking change | Bumped to 0.5.0 in commit `aba663c`; regressed to 0.4.0 by PR #20. **Re-fixed 2026-08-02** (#41): `pyproject.toml` is 0.5.0 again, and ADR-022 rule 5 now states the versioning policy that was previously unwritten. | 2026-04-02 | | C-12 | 4 | `to_dataframe()` requires undeclared pandas optional dependency | Added `pandas = {version = "^1.5.3", optional = true}` and `[tool.poetry.extras] dataframe = ["pandas"]`. | 2026-04-02 | | C-14 | 4 | Editable install metadata stale | Ran `pip install -e ".[dataframe]"` to refresh; went stale again when PR #20 reverted the version. **Re-fixed 2026-08-02** (#41): reinstalled with `[dataframe,frames]`; `importlib.metadata.version` and the MetricFrame `scoring_code_version` stamp both report 0.5.0. | 2026-04-04 | -| C-15 | 4 | `to_dataframe()` is the only remaining dataclass consumer | Accepted as design — `to_dataframe()` owns the dataclass-to-DataFrame path internally. No action needed unless metric catalog grows beyond 30 entries. | 2026-04-04 | +| C-15 | 4 | `to_dataframe()` is the only remaining dataclass consumer | Accepted as design — `to_dataframe()` owns the dataclass-to-DataFrame path internally. No action needed unless metric catalog grows beyond 30 entries. *Corrected 2026-09-18 (S7, #63):* the rationale named the wrong consumer. `to_dataframe()` is gone in 2.0.0 and the dataclasses remain, because their consumer is the public `get_schema_results()`, which was overlooked when this closed. The acceptance stands on that consumer; the FM1 field-mismatch guard still couples `metrics.py` to the catalog. | 2026-04-04 | | C-16 | 3 | Metric function exceptions propagate without context | Wrapped `spec.function()` call in `_calculate_metrics()` with try/except that re-raises as `ValueError` naming the metric, task, and pred_type. Test: `test_metric_function_error_includes_metric_name`. | 2026-04-04 | | C-17 | 4 | `max_allowed_step = 999` hardcoded sentinel | Changed to `float('inf')`. Steps >= 1000 now correctly evaluated. Test: `test_step_values_above_999_not_silently_dropped`. | 2026-04-04 | | C-18 | 4 | No bounds validation on metric hyperparameters | Added bounds validation in `resolve_metric_params()` for `alpha`, `quantile`, `lower_quantile`, `upper_quantile` — all must be in (0, 1). Cross-validation: `lower_quantile < upper_quantile`. 7 tests in `TestResolveMetricParamsBoundsRed`. | 2026-04-04 | @@ -346,6 +388,7 @@ Moved out of the register (no correctness/reliability dimension) — tracked in | C-05 | 3 | sklearn/scipy in pure-math core | `AP` and `MTD` reimplemented in numpy as transcriptions of scikit-learn 1.7.2 (S4, #70: kernels alongside + parity suite, seen red on 26 injected defects; S5, #64: the switch). scikit-learn moved to the dev group as the parity oracle. `TestLevelZeroImportPurity` asserts a fresh `import views_evaluation` loads neither scikit-learn nor pandas and fails (not skips) if either is uninstalled; a CI step proves the runtime-only install. scipy stays (ADR-011 permits it). The empty-group AP case was ruled on the way (D-01, ADR-015 R9). | #70, #64 | | C-44 | 4 | `to_dataframe()` without pandas raises a bare `ModuleNotFoundError`, unlike its guarded sibling | `find_spec("pandas")` gate raising `ModuleNotFoundError(name="pandas")` that names `pip install views-evaluation[dataframe]` — same type as the bare import, so no caller's `except` changes; no log, since Level 0 does not log (standard §5.1). Test `tests/test_adversarial_inputs.py::TestOptionalExtraAbsentRed` (outside the pandas-gated module so it runs where the case applies; observed red on the bare import first). The entry's second cited site, `metrics.py`'s `evaluation_dict_to_dataframe`, is deprecated with the method and deleted in S7 rather than guarded. Same change deprecates the method (S3, #69). | #69 | | C-39 | 2 | `scoring_code_version` stamps the consumer's git SHA when the wheel is installed inside the consumer's checkout | `_source_git_sha` bounded to the directory containing `views_evaluation/` (`parents[2]`, flat layout), accepted only when a `pyproject.toml` there declares this distribution (parsed with `tomllib`, PEP 503-normalised); reads UTF-8; follows a `.git` file's `gitdir:` relative to the file and a linked worktree's `commondir`; rejects non-hash ref contents; logs at WARNING when `.git` is ours but unreadable. Tests `TestSourceGitShaBoundary{Red,Green,Beige}` in `tests/test_metric_frame.py` (consumer venv, vendored copy, unreadable state, loose/packed refs, hyphenated name, linked worktree, submodule); the consumer-venv and relative-`gitdir:` cases were observed failing on the old walker first. CHANGELOG, CIC MetricFrame.md and CIC EvaluationReport.md corrected. Epic #66, story #67; the `/code-review max` of that PR found the `commondir`, locale, hex-check and name-gate gaps and they were fixed before merge. | #67 | +| C-40 | 3 | `to_dataframe()` silently erases any metric that is `nan` in every group | `to_dataframe()`, `BaseEvaluationMetrics.evaluation_dict_to_dataframe()` (the `df.loc[:, df.notna().any()]` filter) and the three unused `make_*_evaluation_dict` factories deleted; pandas moved from the `dataframe` extra (deleted) to the dev group so the import-purity guard keeps something to prove; `test_removed_dataframe_surface_stays_removed` fails on a same-name reintroduction (class or `__getattr__` shim); the package-wide import guard (one allowlist per file, over import statements and string literals handed to import machinery under any alias) fails on a pandas import however it is spelled short of string concatenation. Ships in 2.0.0 after 1.1.0's `DeprecationWarning`; closed 2026-09-18. The remaining two export paths, `to_dict()` and `to_metric_frame()`, agree on the metric set by construction. | #63 | ### Closed by the Fail-Loud Doctrine epic (#26), 2026-08-02 diff --git a/tests/test_adversarial_inputs.py b/tests/test_adversarial_inputs.py index f7aa00f..5594c4e 100644 --- a/tests/test_adversarial_inputs.py +++ b/tests/test_adversarial_inputs.py @@ -2,7 +2,6 @@ import pytest from views_evaluation.evaluation.evaluation_frame import EvaluationFrame -from views_evaluation.evaluation.evaluation_report import EvaluationReport from views_evaluation.evaluation.native_evaluator import NativeEvaluator @@ -128,36 +127,33 @@ def test_inf_rejected_before_metric_executes(self): class TestOptionalExtraAbsentRed: - """Register C-44. A missing optional extra must name itself; this module has no - pandas import-skip, so the case runs wherever the suite runs.""" - - def test_to_dataframe_without_pandas_names_the_extra(self, monkeypatch, caplog): - """Contract, not mechanism: pandas genuinely unimportable (a `None` entry in - sys.modules makes both `import pandas` and `find_spec("pandas")` report absence - on CPython) → the deprecation warning is still emitted first, the raise names - `views-evaluation[dataframe]`, keeps the type the bare import raised - (`ModuleNotFoundError`, `.name == "pandas"`), the extra it names is one - pyproject.toml declares, and nothing is logged (Level 0 does not log). Seen red - on the bare import first: that raised `ModuleNotFoundError: import of pandas - halted` with no extra named.""" + """A missing optional extra must name itself, with the extra pyproject actually + declares, in the exception type the CIC documents, logged on the emit path. Until + 2.0.0 this class pinned `to_dataframe()`'s pandas branch (C-44); that method is gone, + and this is the same shape transplanted to the surviving bridge. Seen red on three + injections before it was trusted: extra renamed to `[framez]`, `ImportError` swapped + for `RuntimeError`, and the `.error()` call deleted — and a fourth, the logger renamed, found by the guard audit.""" + + def test_to_metric_frame_without_views_frames_names_the_extra(self, monkeypatch, caplog): import logging import re import sys import tomllib - import warnings from pathlib import Path + from views_evaluation.evaluation.evaluation_report import EvaluationReport - monkeypatch.setitem(sys.modules, "pandas", None) + # A `None` entry makes both `import views_frames` and `find_spec` report absence. + monkeypatch.setitem(sys.modules, "views_frames", None) report = EvaluationReport('t', 'regression', 'point', { 'month': {'month100': {'MSE': 42.0}}, 'time_series': {}, 'step': {}}) - with caplog.at_level(logging.DEBUG): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - with pytest.raises(ModuleNotFoundError, match=r"views-evaluation\[dataframe\]") as exc: - report.to_dataframe('month') - assert [x.category for x in w] == [DeprecationWarning], "warn first, then raise" - assert exc.value.name == "pandas" - assert not caplog.records, "Level 0 does not log (logging standard §5.1)" + with caplog.at_level(logging.ERROR, logger="views_evaluation.evaluation.metric_frame"): + with pytest.raises(ImportError, match=r"views-evaluation\[frames\]") as exc: + report.to_metric_frame() + errors = [r for r in caplog.records if r.levelno == logging.ERROR] + assert errors and errors[0].message == str(exc.value), "emit path logs the message it raises (§5.1)" + assert errors[0].name == "views_evaluation.evaluation.metric_frame", ( + "the emit path logs under the metric_frame logger, not the file's (§5.1)" + ) # The extra named in the message must exist, or the advice is unfollowable. extra = re.search(r"views-evaluation\[(\w+)\]", str(exc.value)).group(1) pyproject = tomllib.loads((Path(__file__).resolve().parents[1] / "pyproject.toml").read_text(encoding="utf-8")) diff --git a/tests/test_documentation_contracts.py b/tests/test_documentation_contracts.py index 68758a1..8cde229 100644 --- a/tests/test_documentation_contracts.py +++ b/tests/test_documentation_contracts.py @@ -51,6 +51,33 @@ _DELETED_MODULE_PATHS = [p for _, paths in _DELETED_SYMBOLS for p in paths] _DELETED_CLASS_NAMES = [name for name, _ in _DELETED_SYMBOLS] +# Methods removed in 2.0.0 (epic #66, story #63) after the 1.1.0 DeprecationWarning. +# Matched as whole-word tokens — `to_dataframe(`, `to_dataframe ('month')`, a table row +# `` `to_dataframe` ``, the attribute form `EvaluationReport.to_dataframe` all count — in +# docs, CICs, examples AND package docstrings, so a guide that still shows the method as +# a thing to do is a red build: the exact shape of C-29 (a README promising support the +# package lacks), for a method this time. A mention is exempt only when a removal +# framing sits on the same line as it (`_REMOVAL_FRAMING`), NOT +# when the surrounding paragraph merely contains one of `_HISTORICAL_MARKERS` somewhere: +# the guard audit of 2026-09-18 placed `report.to_dataframe('month')` in a paragraph +# whose other sentence said "no longer", and the block-wide exemption let it through. +_DELETED_METHODS = ( + "to_dataframe", + "evaluation_dict_to_dataframe", + "make_time_series_wise_evaluation_dict", + "make_step_wise_evaluation_dict", + "make_month_wise_evaluation_dict", +) +# Word-anchored phrases that predicate removal on the thing named. Ordinary +# collocations (`used to`, `was the`, `deleted`) exempted "is used to build the table" +# and an unanchored `removed` matched `removed_cols = ...` (release review, 2026-09-18); +# `\b` on both sides is what keeps `removed_cols` out (`_` is a word character). +_REMOVAL_FRAMING = re.compile( + r"\b(?:removed|removal|gone in \d|until \d+\.\d|no longer|deprecated|" + r"retired|purged|lost the|left in \d|now raises)\b", + re.I, +) + # Files where a historical mention is legitimate: change logs, decision records and # the risk register all necessarily discuss things that no longer exist. _HISTORICAL_ALLOWLIST = { @@ -249,6 +276,70 @@ def test_no_doc_or_docstring_names_a_deleted_class_as_current(self): + "\n ".join(offenders) ) + def test_no_doc_or_docstring_presents_a_removed_method_as_current(self): + """`to_dataframe()` left in 2.0.0. Eight documents were hand-edited for it; this + is what makes the ninth a red build instead of a C-29 repeat. A mention is exempt + only with a removal framing ON THE SAME LINE, which is how the metrics.py + docstring ("Until 2.0.0 it also carried") and the CIC's Evolution Notes pass; + dated CHANGELOG sections of published releases are frozen and skipped, the + section being cut is scanned. Seen red on + `report.to_dataframe('month')` injected into the integration guide, the + `[Unreleased]` section, a docstring, a CIC, an example, and — after the guard + audit — with a space before the paren, as a bare table token, as an attribute, + and with "no longer" one line away. Known residual: a single LINE that both + frames a removal and advertises the method ("X is no longer shipped; call + to_dataframe()"), or an identifier named `removed`, or a `# deprecated` + trailer, passes; no text heuristic distinguishes those from a legitimate + "to_dataframe() is no longer available".""" + import views_evaluation + pkg_root = Path(views_evaluation.__file__).parent + offenders = [] + # Wider than `_doc_files()`: the CICs' Evolution Notes are framed line by line + # (so the CIC of the very class that lost the method is NOT exempt here), and + # `examples/` is scanned by nothing else. ADRs and reports stay history. + sources = ([README, CHANGELOG] + sorted(DOCS.rglob("*.md")) + sorted(pkg_root.rglob("*.py")) + + sorted((REPO_ROOT / "examples").rglob("*.py")) + sorted((REPO_ROOT / "examples").rglob("*.md"))) + pattern = re.compile(r"\b(" + "|".join(_DELETED_METHODS) + r")\b") + for path in sources: + rel = path.relative_to(REPO_ROOT).as_posix() + if rel.startswith(("documentation/ADRs", "documentation/contributor_protocols")): + continue # history by nature (the CICs are NOT skipped: framed line by line) + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + scan = set(range(len(lines))) + if path == CHANGELOG: + # A dated section of an already-published release is a frozen record and + # legitimately names what it deprecated. `[Unreleased]` AND the section + # for the version pyproject declares — the release being cut, which the + # checklist points consumers at — make current claims and are scanned, + # fenced code included (a fence-stripped body hid a "how to" example in + # the [2.0.0] section; guard audit, 2026-09-18). Headings are located + # with the shared parser's regex on the RAW text so line numbers are real. + TestChangelogCoversTheDeclaredVersion._released_sections(text) # structural asserts + pyproject = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + version = pyproject.get("project", {}).get("version") or pyproject["tool"]["poetry"]["version"] + heading = TestChangelogCoversTheDeclaredVersion._HEADING + starts = [i for i, line in enumerate(lines) if heading.match(line)] + scan = set(range(starts[0] if starts else len(lines))) + for k, i in enumerate(starts): + if heading.match(lines[i]).group("version") == version: + scan |= set(range(i, starts[k + 1] if k + 1 < len(starts) else len(lines))) + for i, line in enumerate(lines): + if i not in scan: + continue + m = pattern.search(line) + if not m: + continue + # Same line only: a one-line-either-side window still admitted a + # "no longer" one sentence away (guard audit, 2026-09-18). + if _REMOVAL_FRAMING.search(line): + continue + offenders.append(f"{rel}:{i + 1} presents `{m.group(1)}` as current: {line.strip()[:80]}") + assert not offenders, ( + "A document, example or docstring presents a method removed in 2.0.0 as current:\n " + + "\n ".join(offenders) + ) + def test_documentation_uses_current_metric_names(self): """Metric tokens in prose must exist in METRIC_CATALOG. @@ -441,20 +532,31 @@ def test_changelog_has_a_section_for_the_pyproject_version(self): class TestLoggingScopeContract: """The Level-0/Level-1 split from logging standard §5.1, asserted against code.""" + # Must agree with logging standard §5.1 and with `TestLevelZeroImportPurity._LEVEL_0` + # (tests/test_metric_calculators.py): `metrics` and `config_schema` were classified + # Level 0 by the import guard on 2026-09-18 and belong here too. _LEVEL_0 = [ "evaluation_frame", "native_evaluator", "evaluation_report", - "metric_catalog", "native_metric_calculators", + "metric_catalog", "native_metric_calculators", "metrics", "config_schema", ] @pytest.mark.parametrize("module_name", _LEVEL_0) def test_level_zero_modules_declare_no_module_logger(self, module_name): import importlib + import logging module = importlib.import_module(f"views_evaluation.evaluation.{module_name}") - src = inspect.getsource(module) - # A module-level `logger = logging.getLogger(__name__)` is the violation. - assert not re.search(r"^logger\s*=\s*logging\.getLogger", src, re.MULTILINE), ( + # Behavioural, not textual: any module-level binding that IS a Logger, whatever + # it is called and however it was obtained (`_logger = ...`, `from logging import + # getLogger`) — a regex on `^logger = logging.getLogger` missed both (guard audit, + # 2026-09-18). The text check stays for the canonical spelling in a docstring. + loggers = sorted(k for k, v in vars(module).items() if isinstance(v, (logging.Logger, logging.LoggerAdapter))) + assert not loggers, ( f"{module_name} is Level 0 and must not maintain a module logger " - f"(logging standard §5.1)" + f"(logging standard §5.1); found {loggers}" + ) + src = inspect.getsource(module) + assert not re.search(r"^\w+\s*=\s*(?:logging\.)?getLogger\(", src, re.MULTILINE), ( + f"{module_name} binds a logger at module level (logging standard §5.1)" ) def test_standard_names_both_levels_explicitly(self): diff --git a/tests/test_evaluation_report.py b/tests/test_evaluation_report.py index 182951e..43e5e6b 100644 --- a/tests/test_evaluation_report.py +++ b/tests/test_evaluation_report.py @@ -2,20 +2,21 @@ Direct unit tests for EvaluationReport. Structured per ADR-020 (Red/Beige/Green): - GREEN — construction, to_dict, get_schema_results, to_dataframe for all four task/type combos - BEIGE — empty schema, multiple metrics per group, raw schema passthrough, the - to_dataframe() DeprecationWarning (one per call, attributed to the caller) - RED — unknown schema key, invalid task/pred_type combination - -The no-pandas fail-loud case for to_dataframe() (C-44) lives in -tests/test_adversarial_inputs.py: this module import-skips on pandas, so it could never -run here in the one environment that case describes. + GREEN — construction, to_dict, get_schema_results for all four task/type combos + BEIGE — empty schema, multiple metrics per group + RED — unknown schema key, invalid task/pred_type combination, the removed + to_dataframe() surface staying removed + +Until 2.0.0 this module import-skipped on pandas for the to_dataframe() tests, so +without the `dataframe` extra every test here was silently skipped. It now runs in +every environment; the report has no pandas surface left. """ +import logging + import pytest -pd = pytest.importorskip("pandas") -from views_evaluation.evaluation.evaluation_report import EvaluationReport # noqa: E402 -from views_evaluation.evaluation.metrics import ( # noqa: E402 +from views_evaluation.evaluation.evaluation_report import EvaluationReport +from views_evaluation.evaluation.metrics import ( RegressionPointEvaluationMetrics, RegressionSampleEvaluationMetrics, ClassificationPointEvaluationMetrics, @@ -97,6 +98,36 @@ def test_get_schema_results_regression_point_returns_correct_cls(self): assert schema['month100'].MSE == 1.0 assert schema['month101'].MSE == 2.0 + @pytest.mark.parametrize("schema, groups", [ + ("month", {"month100": 1.0, "month101": 2.0}), + ("time_series", {"ts00": 1.5}), + ("step", {"step01": 1.0, "step02": 2.0}), + ]) + def test_get_schema_results_maps_every_schema(self, schema, groups): + """All three schemas, not only `month`: until 2.0.0 the `time_series` and `step` + paths were exercised only through the deleted to_dataframe tests, so a + `get_schema_results` that only looked up `month` would have gone unnoticed + (release review, 2026-09-18).""" + report = EvaluationReport('t', 'regression', 'point', _regression_point_results()) + mapped = report.get_schema_results(schema) + assert {k: v.MSE for k, v in mapped.items()} == groups + assert all(isinstance(v, RegressionPointEvaluationMetrics) for v in mapped.values()) + + def test_to_dict_schemas_is_the_live_structure_not_a_copy(self): + """Documented identity (1.0.0 behaviour): `to_dict()['schemas']` IS the internal + dict — the same object across calls and the object `to_metric_frame()` reads. + Pinned so a silent switch to a copy (or a silent switch of the emit path to a + different object) is a red build rather than an invisible behaviour change.""" + results = _regression_point_results() + report = EvaluationReport('t', 'regression', 'point', results) + first = report.to_dict()['schemas'] + assert first is report.to_dict()['schemas'] + assert first is results + first['month']['month100']['MSE'] = 42.0 + mf = report.to_metric_frame() + row = (mf.identifiers["group_id"] == "month100") & (mf.identifiers["metric"] == "MSE") + assert mf.values[row].item() == 42.0, "the emit path reads the same live structure" + def test_get_schema_results_regression_sample_returns_correct_cls(self): report = EvaluationReport('t', 'regression', 'sample', _regression_sample_results()) schema = report.get_schema_results('month') @@ -115,48 +146,8 @@ def test_get_schema_results_classification_sample_returns_correct_cls(self): assert isinstance(schema['month100'], ClassificationSampleEvaluationMetrics) assert schema['month100'].CRPS == 0.3 - def test_to_dataframe_returns_dataframe_with_correct_shape(self): - report = EvaluationReport('t', 'regression', 'point', _regression_point_results()) - with pytest.warns(DeprecationWarning): # deprecated; these tests go in 2.0.0 - df = report.to_dataframe('month') - assert isinstance(df, pd.DataFrame) - assert len(df) == 2 # month100 and month101 - - def test_to_dataframe_index_matches_group_keys(self): - report = EvaluationReport('t', 'regression', 'point', _regression_point_results()) - with pytest.warns(DeprecationWarning): # deprecated; these tests go in 2.0.0 - df = report.to_dataframe('month') - assert set(df.index.tolist()) == {'month100', 'month101'} - - def test_to_dataframe_time_series_schema(self): - report = EvaluationReport('t', 'regression', 'point', _regression_point_results()) - with pytest.warns(DeprecationWarning): # deprecated; these tests go in 2.0.0 - df = report.to_dataframe('time_series') - assert isinstance(df, pd.DataFrame) - assert 'ts00' in df.index - - def test_to_dataframe_step_schema(self): - report = EvaluationReport('t', 'regression', 'point', _regression_point_results()) - with pytest.warns(DeprecationWarning): # deprecated; these tests go in 2.0.0 - df = report.to_dataframe('step') - assert isinstance(df, pd.DataFrame) - assert 'step01' in df.index - - -# --------------------------------------------------------------------------- -# BEIGE: Edge cases -# --------------------------------------------------------------------------- - class TestEvaluationReportBeige: - def test_to_dataframe_empty_schema_returns_empty_dataframe(self): - results = {'month': {}, 'time_series': {}, 'step': {}} - report = EvaluationReport('t', 'regression', 'point', results) - with pytest.warns(DeprecationWarning): # deprecated; these tests go in 2.0.0 - df = report.to_dataframe('month') - assert isinstance(df, pd.DataFrame) - assert len(df) == 0 - def test_get_schema_results_empty_schema_returns_empty_dict(self): results = {'month': {}, 'time_series': {}, 'step': {}} report = EvaluationReport('t', 'regression', 'point', results) @@ -173,54 +164,6 @@ def test_multiple_metrics_per_group_accessible(self): assert schema['month100'].MSE == 1.0 assert schema['month100'].RMSLE == 0.5 - def test_to_dataframe_emits_exactly_one_deprecation_warning_naming_the_replacement(self): - """ADR-022 §2: the warning ships one release before the removal (S3, #69). - `to_dict()` is the replacement and `2.0.0` the removal version; both are in the - text so a consumer reading the warning knows what to do and by when.""" - import warnings - report = EvaluationReport('t', 'regression', 'point', { - 'month': {'month100': {'MSE': 42.0}}, 'time_series': {}, 'step': {}}) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - df = report.to_dataframe('month') - assert len(w) == 1, [str(x.message) for x in w] - assert issubclass(w[0].category, DeprecationWarning) - assert "to_dict()['schemas'][schema]" in str(w[0].message) and "2.0.0" in str(w[0].message) - # stacklevel=2: the warning is attributed to the CALLER's file, which is what - # makes it visible under CPython's default `default::DeprecationWarning:__main__` - # filter for a script consumer. stacklevel=1 would attribute it to the library - # and every default-filtered consumer would see nothing. - assert w[0].filename == __file__ - assert "MSE" in df.columns - - def test_evaluation_dict_to_dataframe_warns_when_called_directly(self): - """The helper is reachable on a dataclass from get_schema_results(); a direct - caller must get the deprecation too (it goes in 2.0.0 with to_dataframe).""" - report = EvaluationReport('t', 'regression', 'point', { - 'month': {'month100': {'MSE': 42.0}}, 'time_series': {}, 'step': {}}) - mapped = report.get_schema_results('month') - cls = type(mapped['month100']) - with pytest.warns(DeprecationWarning, match="2.0.0"): - df = cls.evaluation_dict_to_dataframe(mapped) - assert "MSE" in df.columns - - def test_to_dataframe_raw_returns_internal_results_dict_with_deprecation(self): - """schema='raw' is a deprecated passthrough to the raw results dict.""" - import warnings - results = {'month': {'month100': {'MSE': 42.0}}, 'time_series': {}, 'step': {}} - report = EvaluationReport('t', 'regression', 'point', results) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - raw = report.to_dataframe('raw') - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) - # The raw path's replacement is the whole schemas dict, not one schema. - assert "to_dict()['schemas'] in" in str(w[0].message) - assert 'month' in raw - assert raw['month']['month100']['MSE'] == 42.0 - # Identity, as documented: the same object as to_dict()['schemas'], not a copy. - assert raw is report.to_dict()['schemas'] - def test_all_four_task_pred_type_combinations_resolve_correctly(self): """get_schema_results must not raise for any valid (task, pred_type) pair.""" combos = [ @@ -253,17 +196,57 @@ def test_non_dict_schema_value_fails_at_access(self): with pytest.raises(AttributeError): report.get_schema_results('month') + def test_public_surface_is_exactly_the_documented_one(self): + """2.0.0 removed to_dataframe() and the evaluation_dict_to_dataframe helper after + 1.1.0's DeprecationWarning (ADR-022 §2; register C-40), and the three + make_*_evaluation_dict factories as private cleanup (unused since 0.4.0). Pinned + as an ALLOWLIST of public names (a helper under a new name — `to_records`, + `as_rows` — is a red build and must go through the CIC) PLUS a behavioural probe + of the removed names on instances and a ban on `__getattr__`, because `dir()` + does not list what `__getattr__` serves (release review and guard audit, + 2026-09-18). The import-purity guard separately catches any pandas import + whatever the name.""" + import dataclasses + report = EvaluationReport('t', 'regression', 'point', _regression_point_results()) + + def public(obj): + return sorted(n for n in dir(obj) if not n.startswith("_")) + + removed = ("to_dataframe", "evaluation_dict_to_dataframe", "make_time_series_wise_evaluation_dict", + "make_step_wise_evaluation_dict", "make_month_wise_evaluation_dict") + assert public(EvaluationReport) == ["get_schema_results", "to_dict", "to_metric_frame"] + assert public(report) == ["get_schema_results", "pred_type", "target", "task", "to_dict", "to_metric_frame"] + # `dir()` never lists what a `__getattr__` serves, so the removed names are ALSO + # probed behaviourally on an instance (guard audit, 2026-09-18: a shim survived + # the listing alone), and no `__getattr__` may exist at all. + for name in removed: + assert not hasattr(report, name), f"EvaluationReport().{name} is served (a shim?)" + assert not hasattr(EvaluationReport, "__getattr__") + for cls in (RegressionPointEvaluationMetrics, RegressionSampleEvaluationMetrics, + ClassificationPointEvaluationMetrics, ClassificationSampleEvaluationMetrics): + fields = sorted(f.name for f in dataclasses.fields(cls)) + assert public(cls) == fields, f"{cls.__name__} carries non-field public attributes" + assert public(cls()) == fields, f"{cls.__name__}() serves non-field attributes (a shim?)" + for name in removed: + assert not hasattr(cls(), name), f"{cls.__name__}().{name} is served (a shim?)" + assert not hasattr(cls, "__getattr__") + + def test_metric_with_no_dataclass_field_raises(self, caplog): + """FM1 guard: a computed metric that has no field on the typed dataclass raises + instead of being dropped. The CIC has claimed a Red test for this since before + 2.0.0; there was none, and the guard could be replaced by `continue` unnoticed.""" + report = EvaluationReport('t', 'regression', 'point', { + 'month': {'month100': {'MSE': 1.0, 'BOGUS': 9.9}}, 'time_series': {}, 'step': {}}) + with caplog.at_level(logging.DEBUG): + with pytest.raises(ValueError, match="no field exists"): + report.get_schema_results('month') + assert not caplog.records, "Level 0 does not log (logging standard §5.1); only the emit path does" + def test_get_schema_results_unknown_schema_raises_key_error(self): report = EvaluationReport('t', 'regression', 'point', {}) with pytest.raises(KeyError, match="nonexistent"): report.get_schema_results('nonexistent') - def test_to_dataframe_unknown_schema_raises_key_error(self): - report = EvaluationReport('t', 'regression', 'point', _regression_point_results()) - with pytest.raises(KeyError): - with pytest.warns(DeprecationWarning): - report.to_dataframe('nonexistent_schema') - def test_invalid_task_type_raises_on_get_schema_results(self): """An unrecognised (task, pred_type) pair must fail at lookup time.""" results = {'month': {'m': {'MSE': 1.0}}, 'time_series': {}, 'step': {}} diff --git a/tests/test_falsification_ci_coverage_gap.py b/tests/test_falsification_ci_coverage_gap.py index b126a5a..43683a5 100644 --- a/tests/test_falsification_ci_coverage_gap.py +++ b/tests/test_falsification_ci_coverage_gap.py @@ -23,14 +23,18 @@ (These read 361/68 until 2026-08-02, against a docstring claiming they were "measured, not inferred". They were not: 361 was a later local count that included tests added after the tag, and 68 followed from it. 44 + 22 = 66 is forced by the - two per-file counts on the line above.) + two per-file counts on the line above. Since 2.0.0 `test_evaluation_report.py` no + longer import-skips at all — its pandas surface was removed with `to_dataframe()`. + Executed with `views_frames` made unimportable, two skips remain, and they are what + this guard protects: the module-level skip in `test_metric_frame.py` and the + function-level one in `test_falsification_legacy_compatibility.py`.) This matters more than the raw count. ADR-022 §1 designates the `MetricFrame` on-disk format and axis vocabulary a **cross-repo contract**, "treated as public API regardless of `__all__`". It is the artifact `views-reporting` and `views-pipeline-core` consume. -Its 44 guards, plus the emit path in `test_evaluation_report.py` and the end-to-end probe -in `test_falsification_legacy_compatibility.py`, are exactly the tests that do not run on -any pull request. +Its 44 guards, plus the end-to-end probe in `test_falsification_legacy_compatibility.py`, +were exactly the tests that did not run on any pull request. (`test_evaluation_report.py` +was skipped for pandas, not for `views_frames`; it never held a `to_metric_frame` test.) The skip is silent by construction: a green CI run reporting "293 passed" is indistinguishable from one where the contract is verified. That is register C-37's diff --git a/tests/test_falsification_extras_actually_installed.py b/tests/test_falsification_extras_actually_installed.py index 340c0f3..fe11048 100644 --- a/tests/test_falsification_extras_actually_installed.py +++ b/tests/test_falsification_extras_actually_installed.py @@ -450,10 +450,23 @@ class TestPublishGateIsReal: run: | set -e poetry sync --only main --all-extras - poetry run python -c "import sys, importlib.util as u, views_evaluation, views_evaluation.evaluation.metric_frame; assert u.find_spec('sklearn') is None, 'scikit-learn still installed'; assert 'pandas' not in sys.modules, 'import loaded pandas'" + poetry run python - <<'PY' + import importlib.util as u + import numpy as np + from views_evaluation import EvaluationFrame, NativeEvaluator + assert u.find_spec('sklearn') is None, 'scikit-learn still installed' + assert u.find_spec('pandas') is None, 'pandas still installed' + ids = {k: np.array(v) for k, v in {'time': [1, 1], 'unit': [1, 2], 'origin': [0, 0], 'step': [1, 1]}.items()} + cfg = {'steps': [1], 'classification_targets': ['t'], 'classification_point_metrics': ['AP', 'Brier_cls_point']} + report = NativeEvaluator(cfg).evaluate(EvaluationFrame(np.array([0., 1.]), np.array([[0.2], [0.8]]), ids, metadata={'target': 't'})) + mf = report.to_metric_frame(model_id='ci', run_type='test', timestamp=1758153600, evaluation_timestamp='2026-09-18T00:00:00') + assert isinstance(mf.metadata.provenance.timestamp, int), 'timestamp is int in the cross-repo contract (C-47)' + assert mf.n_rows == len(cfg['classification_point_metrics']) * 2 * 3, mf.n_rows # metrics x (group + mean) x schemas + print('runtime-only install: evaluate + to_metric_frame OK, rows =', mf.n_rows) + PY poetry install --all-extras - name: Verify optional extras are installed - run: poetry run python -c "import views_frames, pandas" + run: poetry run python -c "import views_frames" - name: Run tests run: | set -e diff --git a/tests/test_metric_calculators.py b/tests/test_metric_calculators.py index 91d15fd..b1be561 100644 --- a/tests/test_metric_calculators.py +++ b/tests/test_metric_calculators.py @@ -1814,111 +1814,251 @@ class TestLevelZeroImportPurity: pandas in `sys.modules` — the "zero knowledge of external data frameworks" claim was true of the source and false of the interpreter (C-05). - Three guards, because an audit showed two half-guards leave a seam. The static one - walks every import statement — top-level or lazy — in each Level-0 module, requires - its root to be allowed, forbids importing the Level-1 module (`metric_frame`) from - the pure modules, and forbids dynamic-import machinery (`__import__`, - `importlib.import_module`, `exec`) outright, since a call-time import spelled as a - call is invisible to an AST walk over import statements. The dynamic one imports the - package in a fresh interpreter AND exercises a kernel of each kind and one full - evaluation, then asserts that neither pandas nor scikit-learn was loaded; it is + Three kinds of guard, because an audit showed two half-guards leave a seam. The + static one walks every file in the package and every import in it — statement or + string literal handed to import machinery, top-level or lazy — and requires each + root to be on that file's allowlist; Level 0 files may name only numpy and scipy + and may not import the Level-1 module; `exec`/`eval` and non-literal machinery + calls are forbidden everywhere. The dynamic one imports the package in a fresh + interpreter AND exercises a kernel of each kind, one full evaluation, one emit and + one save/load, then asserts that neither pandas nor scikit-learn was loaded; it is meaningful only if both are installed and not loaded, so it fails, not skips, when - they are missing. The third pins the declaration: scikit-learn is in the dev group - only. + they are missing. The third pins the declarations: the runtime dependency set and + the extras, exactly, under either pyproject layout. """ - _PURE_LEVEL_0 = ["evaluation_frame", "native_evaluator", "metric_catalog", "native_metric_calculators"] - # ADR-011 §Layering: "No external imports except numpy and scipy". The package's own - # modules are allowed by name below, not by root, so intra-package topology is checked. + # ONE declaration, keyed by path relative to the package: what each file may import + # from outside the standard library and this package, statically or through import + # machinery. Set to the measured census (2026-09-18) — no dead allowances, so a + # module that stops importing scipy must drop it here too. Level 0 modules may name + # only numpy and scipy (ADR-011; pinned by `test_the_allowlist_is_adr_011s`); the + # two Level-1 bridge modules add views_frames (the root `__init__` only PROBES for it + # with `find_spec`, which imports nothing). Every other file gets nothing. An allowlist, not a + # denylist: `import xarray` in metric_frame.py is as wrong as `import pandas`, and a + # denylist can only name the frameworks it has met. Keyed by path, not stem: a stem + # key `__init__` gave `profiles/__init__.py` the root package's allowance (release + # review, 2026-09-18). + _ALLOWED_BY_PATH = { + "evaluation/evaluation_frame.py": {"numpy"}, + "evaluation/native_evaluator.py": {"numpy"}, + "evaluation/native_metric_calculators.py": {"numpy", "scipy"}, + "evaluation/evaluation_report.py": {"numpy", "views_frames"}, + "evaluation/metric_frame.py": {"numpy", "views_frames"}, + } + # ADR-011 §Layering: "No external imports except numpy and scipy" for Level 0. _ALLOWED_EXTERNAL = {"numpy", "scipy"} + _LEVEL_0 = ["evaluation/evaluation_frame.py", "evaluation/native_evaluator.py", + "evaluation/metric_catalog.py", "evaluation/native_metric_calculators.py", + "evaluation/metrics.py", "evaluation/config_schema.py"] _LEVEL_1_MODULES = {"metric_frame"} - # evaluation_report.py is Level 0 by the logging standard but carries two lazy - # Level-1 bridges by design (`to_dataframe` → pandas, deprecated and gone in 2.0.0; - # `to_metric_frame` → views_frames and metric_frame). Held to the same rule plus those. - _BRIDGE_EXTERNAL = {"pandas", "views_frames"} + # The import machinery, by the names it is bound to. A string literal handed to one + # of these (positionally or as `name=`) is an import. Only these callees are read — + # matching a literal under ANY call made `float("nan")` an import of a `nan` package + # whenever one was installed or a `nan/` directory sat on sys.path (release review, + # 2026-09-18). Aliases (`from importlib import import_module as _load`) are resolved + # per file. `exec`/`eval` are forbidden everywhere: an import spelled inside a string + # is invisible to any AST walk. + _IMPORT_MACHINERY = {"__import__", "import_module"} + _FORBIDDEN_CALLS = {"exec", "eval"} @staticmethod - def _imports_of(module_name): - """(external roots, internal module names, dynamic-import call names) for a module.""" + def _package_root(): + from pathlib import Path + import views_evaluation + return Path(views_evaluation.__file__).parent + + @classmethod + def _imports_of(cls, rel_path): + """(external roots, internal module names, dynamic-import call names) for the + package file at `rel_path`.""" + return cls._imports_in((cls._package_root() / rel_path).read_text(encoding="utf-8")) + + @classmethod + def _imports_in(cls, source): + """The same, from source text: import statements at any depth, plus string + literals passed to import machinery (by any local alias), plus the names of + forbidden calls.""" import ast - import importlib - import inspect - - module = importlib.import_module(f"views_evaluation.evaluation.{module_name}") - tree = ast.parse(inspect.getsource(module)) + tree = ast.parse(source) + machinery = set(cls._IMPORT_MACHINERY) + for node in ast.walk(tree): # local aliases of the machinery + if isinstance(node, ast.ImportFrom) and node.module == "importlib": + machinery |= {a.asname or a.name for a in node.names if a.name in cls._IMPORT_MACHINERY} external, internal, dynamic = set(), set(), set() + + def root_of(dotted): + head = dotted.split(".")[0] + (internal if head == "views_evaluation" else external).add( + dotted.split(".")[-1] if head == "views_evaluation" else head) + for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: - root = alias.name.split(".")[0] - (internal if root == "views_evaluation" else external).add(alias.name.split(".")[-1] if root == "views_evaluation" else root) + root_of(alias.name) elif isinstance(node, ast.ImportFrom): if node.level > 0 or (node.module or "").split(".")[0] == "views_evaluation": internal.add((node.module or "").split(".")[-1] or "") - internal |= {alias.name for alias in node.names} if node.level > 0 else set() + # The imported NAMES too, for absolute imports as well as relative: + # `from views_evaluation.evaluation import metric_frame` recorded + # only `evaluation` and the Level-1 crossing check never saw + # `metric_frame` (guard audit, 2026-09-18). + internal |= {alias.name for alias in node.names} elif node.module: external.add(node.module.split(".")[0]) elif isinstance(node, ast.Call): fn = node.func name = fn.id if isinstance(fn, ast.Name) else (fn.attr if isinstance(fn, ast.Attribute) else None) - if name in {"__import__", "import_module", "exec", "eval"}: + if name in cls._FORBIDDEN_CALLS: dynamic.add(name) + elif name in machinery: + literal = next((a for a in node.args[:1]), None) or next( + (kw.value for kw in node.keywords if kw.arg == "name"), None) + if isinstance(literal, ast.Constant) and isinstance(literal.value, str): + root_of(literal.value) + dynamic.add(name) + else: + dynamic.add(f"{name}()") return external, internal, dynamic - @pytest.mark.parametrize("module_name", _PURE_LEVEL_0 + ["evaluation_report"]) - def test_every_import_in_a_level_zero_module_is_allowed(self, module_name): + @pytest.mark.parametrize("rel_path", sorted(str(p) for p in []) or [ + "__init__.py", "adapters/__init__.py", "evaluation/__init__.py", + "evaluation/config_schema.py", "evaluation/evaluation_frame.py", + "evaluation/evaluation_report.py", "evaluation/metric_catalog.py", + "evaluation/metric_frame.py", "evaluation/metrics.py", "evaluation/native_evaluator.py", + "evaluation/native_metric_calculators.py", "profiles/__init__.py", "profiles/base.py", + "profiles/hydranet_ucdp.py", + ]) + def test_every_import_in_the_package_is_on_its_files_allowlist(self, rel_path): + """Every file, static and machinery imports alike. Seen red (2026-09-18) on: + `import pandas` in metrics.py and profiles/base.py; `from importlib import + import_module as _load; _load("pandas")`; `importlib.import_module("pandas")` + and `import xarray` in metric_frame.py; `__import__("pandas")` in + profiles/base.py; `import_module(name="pandas")`; a hard `import views_frames` + in profiles/__init__.py; and `exec("import pandas")` anywhere. A statically + allowed root spelled through machinery is still red: the package imports by + statement only.""" import sys stdlib = sys.stdlib_module_names - external, internal, dynamic = self._imports_of(module_name) - allowed = self._ALLOWED_EXTERNAL | (self._BRIDGE_EXTERNAL if module_name == "evaluation_report" else set()) + external, internal, dynamic = self._imports_of(rel_path) + allowed = self._ALLOWED_BY_PATH.get(rel_path, set()) offending = sorted(r for r in external if r not in stdlib and r not in allowed) assert not offending, ( - f"{module_name} imports {offending}; Level 0 may import only the standard " - f"library, numpy and scipy (ADR-011; C-05)" + f"{rel_path} imports {offending}, which its allowlist does not name " + f"(ADR-011; C-05, C-40). Level 0 may import only the standard library, numpy and scipy." ) - if module_name != "evaluation_report": + if rel_path in self._LEVEL_0: + assert allowed <= self._ALLOWED_EXTERNAL, f"{rel_path} is Level 0 (ADR-011: numpy and scipy only)" crossing = sorted(internal & self._LEVEL_1_MODULES) assert not crossing, ( - f"{module_name} imports Level-1 {crossing}; Level 0 must not depend on the " + f"{rel_path} imports Level-1 {crossing}; Level 0 must not depend on the " f"emit layer (ADR-011 layering) — that would also make the core unimportable " f"without the `frames` extra" ) + forbidden = sorted(d for d in dynamic if d in self._FORBIDDEN_CALLS or d.endswith("()")) + assert not forbidden, ( + f"{rel_path} uses {forbidden}; an import spelled as a string or built at run time " + f"is invisible to this guard and forbidden anywhere in the package" + ) assert not dynamic, ( - f"{module_name} uses {sorted(dynamic)}; dynamic imports are invisible to this " - f"guard and forbidden in Level 0" + f"{rel_path} uses import machinery {sorted(dynamic)}; the package imports by statement " + f"only (`find_spec` is a probe, not machinery, and is allowed)" ) - def test_the_static_guard_is_not_vacuous(self): - """It must see numpy (an `import`) AND scipy (an `import from`) in the kernels, - or one of its two branches is scanning nothing.""" - external, internal, _ = self._imports_of("native_metric_calculators") - assert {"numpy", "scipy"} <= external - external, internal, _ = self._imports_of("native_evaluator") + def test_the_parametrisation_lists_every_package_file(self): + """Vacuity: a new module must appear above or it is guarded by nothing.""" + actual = sorted(p.relative_to(self._package_root()).as_posix() for p in self._package_root().rglob("*.py")) + listed = sorted(self.test_every_import_in_the_package_is_on_its_files_allowlist.pytestmark[0].args[1]) + assert actual == listed, f"package files {actual} vs parametrised {listed}" + + def test_the_allowlist_matches_the_census_exactly(self): + """Positive control and dead-allowance check in one: the set of files that import + anything external, as the walker sees them, must equal the allowlist's keys, and + each file's measured roots must equal its allowance — so the walker is proven to + see numpy (an `import`) and scipy and views_frames (`import from`), and an + allowance nothing uses cannot linger.""" + import sys + stdlib = sys.stdlib_module_names + measured = {} + for path in self._package_root().rglob("*.py"): + rel = path.relative_to(self._package_root()).as_posix() + external, _, _ = self._imports_of(rel) + ext = {r for r in external if r not in stdlib} + if ext: + measured[rel] = ext + assert measured == self._ALLOWED_BY_PATH, (measured, self._ALLOWED_BY_PATH) + assert measured["evaluation/native_metric_calculators.py"] == {"numpy", "scipy"} + _, internal, _ = self._imports_of("evaluation/native_evaluator.py") assert "evaluation_frame" in internal, "the internal-import branch sees nothing" + def test_the_machinery_branch_sees_every_spelling_it_claims_to(self): + """Positive control for the branch no package file exercises today: an aliased + `import_module`, a keyword `name=`, a dotted `importlib.import_module`, a bare + `__import__`, a non-literal argument, and `exec`.""" + external, _, dynamic = self._imports_in( + "import importlib\n" + "from importlib import import_module as _load\n" + "def f(n):\n" + " _load('pandas'); __import__(name='polars'); importlib.import_module('sklearn.metrics')\n" + " import_module(n); exec('import modin')\n" + ) + assert external - {"importlib"} == {"pandas", "polars", "sklearn"}, external + assert {"exec", "import_module()"} <= dynamic, dynamic + def test_the_allowlist_is_adr_011s(self): - """The declaration is the guard; an edit here must be a visible two-place change.""" + """The declaration is the guard; an edit here must be a visible change.""" assert self._ALLOWED_EXTERNAL == {"numpy", "scipy"}, "ADR-011: numpy and scipy only" - def test_scikit_learn_is_declared_dev_only(self): + def test_runtime_dependencies_are_exactly_the_declared_set(self): + """The declaration is the guard: the hard runtime set is numpy and scipy, the + declared set adds the optional views-frames — nothing else, by name, under either + pyproject layout (a hard views-frames in a PEP 621 `[project] dependencies` list + survived a check that only looked at the poetry table; guard audit, 2026-09-18). scikit-learn + (C-05) and pandas (C-40) live in the dev group, where the parity suite and the + purity guard need them present. Re-adding either to runtime, as a hard dependency + or inside an extra, is a red build here rather than a post-push CI probe.""" + import re import tomllib from pathlib import Path - data = tomllib.loads((Path(__file__).resolve().parents[1] / "pyproject.toml").read_text(encoding="utf-8")) - runtime = data["tool"]["poetry"]["dependencies"] - dev = data["tool"]["poetry"]["group"]["dev"]["dependencies"] - assert "scikit-learn" not in runtime, "scikit-learn is back in the runtime dependencies (C-05)" - assert "scikit-learn" in dev, "scikit-learn must stay as the dev-only parity oracle" + from tests.test_falsification_extras_actually_installed import _extras_table + text = (Path(__file__).resolve().parents[1] / "pyproject.toml").read_text(encoding="utf-8") + data = tomllib.loads(text) + poetry = data["tool"]["poetry"] + + def dist_name(spec): # "views-frames[frames]>=1.10,<2 ; python>='3.11'" -> "views-frames" + return re.split(r"[\s\[<>=!~;@]", spec.strip(), 1)[0].lower() + + # Hard runtime dependencies, under either layout. Poetry-table entries are hard + # unless `optional = true`; a PEP 621 `[project] dependencies` entry is always + # hard (an optional one belongs under optional-dependencies). views-frames must + # be optional wherever it is declared. + poetry_deps = {k: v for k, v in poetry.get("dependencies", {}).items() if k != "python"} + hard = {k for k, v in poetry_deps.items() if not (isinstance(v, dict) and v.get("optional") is True)} + hard |= {dist_name(d) for d in data.get("project", {}).get("dependencies", [])} + assert hard == {"numpy", "scipy"}, f"hard runtime dependencies must be numpy and scipy only; got {sorted(hard)}" + declared = set(poetry_deps) | {dist_name(d) for d in data.get("project", {}).get("dependencies", [])} + assert declared == {"numpy", "scipy", "views-frames"}, sorted(declared) + # Extras: names AND members, version specifiers stripped, from whichever table + # is authoritative (the shared parser reads PEP 621 first, as poetry does). + extras = {m.group(1): sorted(dist_name(d) for d in re.findall(r"""["']([^"']+)["']""", m.group(2))) + for m in re.finditer(r'^(\w[\w-]*)\s*=\s*\[([^\]]*)\]', _extras_table(text), re.M)} + assert extras == {"frames": ["views-frames"]}, extras + dev = poetry["group"]["dev"]["dependencies"] + for dist, why in (("scikit-learn", "the dev-only parity oracle (C-05)"), + ("pandas", "what the purity guard proves is not loaded (C-40)")): + assert dist in dev, f"{dist} must stay in the dev group as {why}" def test_importing_and_exercising_the_package_loads_neither_pandas_nor_sklearn(self): """Import-time AND call-time: a kernel that imported pandas only when called would - pass an import-only probe.""" + pass an import-only probe, and so would a `MetricFrame.save()` that did — so + every implemented kernel in the catalog, one emit per cell, and one save/load + are exercised, and the script asserts nothing in the catalog was skipped.""" import importlib.util import os import subprocess import sys from pathlib import Path - for dist, how in (("pandas", "the `dataframe` extra"), ("sklearn", "the dev group"), + for dist, how in (("pandas", "the dev group"), ("sklearn", "the dev group"), ("views_frames", "the `frames` extra")): assert importlib.util.find_spec(dist) is not None, ( f"{dist} is not installed, so this guard cannot prove it is not loaded; " @@ -1930,12 +2070,28 @@ def test_importing_and_exercising_the_package_loads_neither_pandas_nor_sklearn(s import sys, numpy as np import views_evaluation, views_evaluation.evaluation.metric_frame from views_evaluation import EvaluationFrame, NativeEvaluator -from views_evaluation.evaluation.native_metric_calculators import calculate_ap_native, calculate_mtd_native -calculate_ap_native(np.array([1., 0., 1.]), np.array([[.9], [.1], [.8]])) -calculate_mtd_native(np.array([1., 2., 3.]), np.array([[1.5], [2.5], [2.5]]), power=1.5) -ids = {'time': np.array([1, 1]), 'unit': np.array([1, 2]), 'origin': np.array([0, 0]), 'step': np.array([1, 1])} -ef = EvaluationFrame(np.array([0., 1.]), np.array([[.1], [.9]]), ids, metadata={'target': 't'}) -NativeEvaluator({'steps': [1], 'regression_targets': ['t'], 'regression_point_metrics': ['MSE', 'MTD']}).evaluate(ef).to_metric_frame() +from views_evaluation.evaluation.metric_catalog import METRIC_CATALOG, METRIC_MEMBERSHIP +# EVERY implemented kernel, through the evaluator, in all four cells: a lazy import the +# static walker cannot see (a getattr'd import_module, a module loaded by spec) is +# caught only on a path this probe walks, so it walks them all (guard audit, 2026-09-18). +rs = np.random.RandomState(0) +n, S = 8, 5 +ids = {'time': np.repeat([1, 2], 4), 'unit': np.tile([1, 2, 3, 4], 2), 'origin': np.zeros(n, int), 'step': np.repeat([1, 2], 4)} +ran = set() +for (task, ptype), names in METRIC_MEMBERSHIP.items(): + names = sorted(m for m in names if METRIC_CATALOG[m].implemented) + y = rs.randint(0, 2, n).astype(float) if task == 'classification' else rs.gamma(2.0, 1.0, n) + 0.1 + cols = S if ptype == 'sample' else 1 + pred = rs.uniform(0.05, 0.95, (n, cols)) if task == 'classification' else rs.gamma(2.0, 1.0, (n, cols)) + 0.1 + cfg = {'steps': [1, 2], f'{task}_targets': ['t'], f'{task}_{ptype}_metrics': names} + mf = NativeEvaluator(cfg).evaluate(EvaluationFrame(y, pred, ids, metadata={'target': 't'})).to_metric_frame() + ran |= set(names) +missing = sorted({m for m, spec in METRIC_CATALOG.items() if spec.implemented} - ran) +assert not missing, f'kernels the sweep never ran: {missing}' +import tempfile, pathlib +from views_evaluation.evaluation.metric_frame import MetricFrame +with tempfile.TemporaryDirectory() as d: + mf.save(pathlib.Path(d) / 'mf'); MetricFrame.load(pathlib.Path(d) / 'mf') print(sorted(m for m in ('pandas', 'sklearn') if m in sys.modules)) """ out = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, cwd=repo, env=env) diff --git a/views_evaluation/evaluation/evaluation_report.py b/views_evaluation/evaluation/evaluation_report.py index 192918c..501758b 100644 --- a/views_evaluation/evaluation/evaluation_report.py +++ b/views_evaluation/evaluation/evaluation_report.py @@ -1,5 +1,4 @@ import dataclasses -import warnings from typing import Dict, Any, Optional from views_evaluation.evaluation.metrics import ( RegressionPointEvaluationMetrics, @@ -13,7 +12,9 @@ class EvaluationReport: A structured, framework-agnostic container for evaluation results. This class decouples the raw result data from its final presentation - format, allowing for flexible export to JSON, Dictionaries, or Pandas. + format: ``to_dict()`` for JSON and dictionaries, ``to_metric_frame()`` for the + typed evaluation-of-record. It returns no DataFrame; a caller that wants one + builds it from ``to_dict()['schemas'][schema]``. """ def __init__(self, target: str, task: str, pred_type: str, results: Dict[str, Dict[str, Any]]): self.target = target @@ -60,60 +61,6 @@ def get_schema_results(self, schema: str) -> Dict[str, Any]: return mapped_results - def to_dataframe(self, schema: str): - """ - Converts a specific schema's results into a Pandas DataFrame. - If schema='raw', returns the internal results dict — the same object as - ``to_dict()['schemas']``; do not mutate it. - - **Deprecated; removed in 2.0.0** (ADR-022 §2; register C-40, C-44). Build a - DataFrame in the caller from ``to_dict()['schemas'][schema]`` instead: - ``pd.DataFrame.from_dict(report.to_dict()['schemas'][schema], orient='index')`` - carries the same values. It is not byte-identical: this method orders columns by - dataclass field, drops any column that is NaN in every group (C-40), and keeps a - group with no metrics as a NaN row; the recipe orders columns as the metrics were - configured, keeps every column, and omits an empty group. - """ - # One warning on every call, whatever the schema: the `raw` passthrough used to - # carry its own; it is folded in here so a caller sees exactly one. - replacement = "to_dict()['schemas']" if schema == "raw" else "to_dict()['schemas'][schema]" - warnings.warn( - f"EvaluationReport.to_dataframe() is deprecated and will be removed in 2.0.0. " - f"Build a DataFrame from {replacement} in the caller.", - DeprecationWarning, - stacklevel=2, - ) - if schema == "raw": - return self._results - - # Gate on find_spec so the helpful error fires only when the extra is truly - # absent; a genuine import error inside pandas then propagates loudly. Until - # 2026-09-17 this was a bare `import pandas` that surfaced as - # `ModuleNotFoundError: No module named 'pandas'` with no mention of the extra - # (register C-44). Raised as ModuleNotFoundError — the type the bare import - # raised — so no `except ModuleNotFoundError` caller changes behaviour (ADR-022 - # §1 counts raised types as public surface). No log: this path computes a value - # in memory and Level 0 does not log (logging standard §5.1); only the - # `to_metric_frame()` emit path below does. - import importlib.util - if importlib.util.find_spec("pandas") is None: - raise ModuleNotFoundError( - "EvaluationReport.to_dataframe() requires the optional 'pandas' " - "dependency. Install it with: pip install views-evaluation[dataframe]", - name="pandas", - ) - import pandas as pd - mapped_results = self.get_schema_results(schema) - if not mapped_results: - return pd.DataFrame() - - metrics_cls = self._get_metrics_cls() - # The helper warns on its own for direct callers; this call already has. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return metrics_cls.evaluation_dict_to_dataframe(mapped_results) - - def to_metric_frame( self, *, @@ -133,8 +80,7 @@ def to_metric_frame( A Level-1 bridge: it flattens the nested per-group results into rows keyed by ``(eval_type, target, metric, group_id, partition, level)`` and attaches provenance. - ``to_dict()`` is unaffected — this is purely additive. (``to_dataframe()`` is - deprecated and goes in 2.0.0.) + ``to_dict()`` is unaffected — this is purely additive. For each schema (month/time_series/step) present, one row is emitted per (group_id, metric), PLUS a cross-group aggregate row with ``group_id="mean"`` carrying @@ -283,7 +229,14 @@ def _emit(eval_type: str, metric: str, group_id: str, value: float) -> None: return MetricFrame(values=values_arr, identifiers=identifiers, metadata=metadata) def to_dict(self) -> Dict[str, Any]: - """Converts the entire report into a nested dictionary.""" + """The report as a nested dictionary. + + ``schemas`` is the report's LIVE internal structure — the same object every call + returns, the object ``to_metric_frame()`` reads, and (as constructed) the caller's + own ``results`` argument — not a copy. Mutating it changes what is subsequently + emitted. That has been the behaviour since 1.0.0 and is pinned by a test; a copy + would be a behaviour change and is not made here. Treat the result as read-only. + """ return { "target": self.target, "task": self.task, diff --git a/views_evaluation/evaluation/metrics.py b/views_evaluation/evaluation/metrics.py index 2e7abce..fe3b2c5 100644 --- a/views_evaluation/evaluation/metrics.py +++ b/views_evaluation/evaluation/metrics.py @@ -4,109 +4,13 @@ @dataclass class BaseEvaluationMetrics: - """ - A data class for basic evaluation methods shared by all evaluation metrics classes. - """ - @classmethod - def make_time_series_wise_evaluation_dict(cls, time_series_length: int =12) -> dict: - """ - Generates a dictionary of EvaluationMetrics instances for a specified number of time series. - - This method facilitates the batch creation of metric containers for multiple time series, initializing them with None. - - Args: - time_series_length (int): The number of time series for which to generate evaluation metrics. Defaults to 12. - - Returns: - dict: A dictionary where each key is a step label (e.g., 'ts01', 'ts02', ...) and each value is an instance of EvaluationMetrics. - - Example: - >>> from utils_evaluation_metrics import EvaluationMetrics - >>> evaluation_dict = EvaluationMetrics.make_evaluation_dict(time_series_length=12) - >>> evaluation_dict['ts01'].MSE = sklearn.metrics.mean_squared_error(ts01_y_true, ts01_y_pred) - >>> evaluation_dict['ts02'].MSE = sklearn.metrics.mean_squared_error(ts02_y_true, ts02_y_pred) - >>> ... - - """ - return {f"ts{str(i).zfill(2)}": cls() for i in range(0, time_series_length)} - - @classmethod - def make_step_wise_evaluation_dict(cls, steps: int =36) -> dict: - """ - Generates a dictionary of EvaluationMetrics instances for a specified number of steps. - - This method facilitates the batch creation of metric containers for multiple steps, initializing them with None. - - Args: - steps (int): The number of forecasting steps for which to generate evaluation metrics. Defaults to 36. - - Returns: - dict: A dictionary where each key is a step label (e.g., 'step01', 'step02', ...) and each value is an instance of EvaluationMetrics. - - Example: - >>> from utils_evaluation_metrics import EvaluationMetrics - >>> evaluation_dict = EvaluationMetrics.make_evaluation_dict(steps=36) - >>> evaluation_dict['step01'].MSE = sklearn.metrics.mean_squared_error(step01_y_true, step01_y_pred) - >>> evaluation_dict['step02'].MSE = sklearn.metrics.mean_squared_error(step02_y_true, step02_y_pred) - >>> ... - - """ - return {f"step{str(i).zfill(2)}": cls() for i in range(1, steps + 1)} - - @classmethod - def make_month_wise_evaluation_dict(cls, month_start: int, month_end: int) -> dict: - """ - Generates a dictionary of EvaluationMetrics instances for a specified range of months. + """Common base of the four typed metric containers below. - This method facilitates the batch creation of metric containers for multiple months, initializing them with None. - - Args: - month_start (int): The first month for which to generate evaluation metrics. - month_end (int): The last month for which to generate evaluation metrics. - - Returns: - dict: A dictionary where each key is a step label (e.g., 'month501', 'month502', ...) and each value is an instance of EvaluationMetrics. - - Example: - >>> from utils_evaluation_metrics import EvaluationMetrics - >>> evaluation_dict = EvaluationMetrics.make_evaluation_dict(month_start=501, month_end=548) - >>> evaluation_dict['month501'].MSE = sklearn.metrics.mean_squared_error(month501_y_true, month501_y_pred) - >>> evaluation_dict['month502'].MSE = sklearn.metrics.mean_squared_error(month502_y_true, month502_y_pred) - >>> ... - - """ - return {f"month{str(i)}": cls() for i in range(month_start, month_end + 1)} - - @staticmethod - def evaluation_dict_to_dataframe(evaluation_dict: dict): - """ - Converts a dictionary of EvaluationMetrics instances into a pandas DataFrame for easy analysis. - - This static method transforms a structured dictionary of evaluation metrics into a DataFrame, where each row corresponds to a forecasting step and columns represent different metrics. - - Args: - evaluation_dict (dict): A dictionary of EvaluationMetrics instances, typically generated by the make_evaluation_dict class method. - - Returns: - pd.DataFrame: A pandas DataFrame where each row indexes a forecasting step and columns correspond to the various metrics stored in EvaluationMetrics. - - Example: - >>> evaluation_df = EvaluationMetrics.evaluation_dict_to_dataframe(evaluation_dict) - - """ - # Deprecated with EvaluationReport.to_dataframe(); removed in 2.0.0 (epic #66). - # Reachable directly on a dataclass returned by get_schema_results(), so it warns - # too. The import stays bare: the only guarded entry is to_dataframe(). - import warnings - warnings.warn( - "evaluation_dict_to_dataframe() is deprecated and will be removed in 2.0.0. " - "Build a DataFrame from EvaluationReport.to_dict()['schemas'][schema] instead.", - DeprecationWarning, - stacklevel=2, - ) - import pandas as pd - df = pd.DataFrame.from_dict(evaluation_dict, orient='index') - return df.loc[:, df.notna().any()] + Until 2.0.0 it also carried three ``make_*_evaluation_dict`` factories (no callers) + and ``evaluation_dict_to_dataframe`` (the DataFrame bridge, removed with + ``EvaluationReport.to_dataframe()``, both gone in 2.0.0; register C-40). The + containers are what ``EvaluationReport.get_schema_results()`` returns. + """ # --------------------------------------------------------------------------- diff --git a/views_evaluation/evaluation/native_metric_calculators.py b/views_evaluation/evaluation/native_metric_calculators.py index 809f2d1..c20c662 100644 --- a/views_evaluation/evaluation/native_metric_calculators.py +++ b/views_evaluation/evaluation/native_metric_calculators.py @@ -76,8 +76,8 @@ def _crps_ensemble_numpy(y_true: np.ndarray, y_pred: np.ndarray) -> np.ndarray: # where scikit-learn returned 0.0 (ADR-015 R9) — and the parity suite in # tests/test_metric_calculators.py still runs them against scikit-learn, now a dev-only # dependency, as the oracle, permanently. Level 0 imports -# numpy and scipy only (ADR-011), so `import views_evaluation` no longer loads pandas -# through scikit-learn. +# numpy and scipy only (ADR-011), so `import views_evaluation` no longer loads a +# dataframe library through scikit-learn. # # Input validation is the kernels' own. On the NativeEvaluator path EvaluationFrame # already rejects NaN, inf and object dtype, but the public kernels and