From 8fe2721e809511407723ef379f7494a380f768d5 Mon Sep 17 00:00:00 2001 From: aminemanai2003 Date: Wed, 12 Aug 2026 16:49:35 +0100 Subject: [PATCH 1/2] Implement ActEval 1.0 roadmap --- .github/workflows/ci.yml | 4 +- .github/workflows/release.yml | 2 +- CHANGELOG.md | 25 ++ CITATION.cff | 13 + CONTRIBUTING.md | 5 + README.md | 418 +++++++++--------- SECURITY.md | 15 + docs/api.md | 31 ++ docs/inference.md | 58 +++ docs/migration-1.0.md | 39 ++ docs/monitoring.md | 40 ++ docs/plan-audit.md | 24 ++ docs/readme-audit.md | 32 ++ docs/reporting.md | 31 ++ docs/stability.md | 34 ++ examples/inference_and_monitoring.py | 67 +++ pyproject.toml | 10 +- src/acteval/__init__.py | 56 ++- src/acteval/_version.py | 3 + src/acteval/api.py | 20 +- src/acteval/inference.py | 605 +++++++++++++++++++++++++++ src/acteval/monitoring.py | 480 +++++++++++++++++++++ src/acteval/reporting.py | 125 ++++++ src/acteval/reports/comparison.py | 17 + src/acteval/reports/result.py | 15 + tests/test_api.py | 22 + tests/test_examples.py | 12 + tests/test_inference.py | 226 ++++++++++ tests/test_monitoring.py | 133 ++++++ tests/test_reporting.py | 55 +++ 30 files changed, 2396 insertions(+), 221 deletions(-) create mode 100644 CITATION.cff create mode 100644 SECURITY.md create mode 100644 docs/inference.md create mode 100644 docs/migration-1.0.md create mode 100644 docs/monitoring.md create mode 100644 docs/readme-audit.md create mode 100644 docs/reporting.md create mode 100644 docs/stability.md create mode 100644 examples/inference_and_monitoring.py create mode 100644 src/acteval/_version.py create mode 100644 src/acteval/inference.py create mode 100644 src/acteval/monitoring.py create mode 100644 src/acteval/reporting.py create mode 100644 tests/test_inference.py create mode 100644 tests/test_monitoring.py create mode 100644 tests/test_reporting.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faba74c..cd1d25b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 @@ -36,4 +36,4 @@ jobs: - run: >- python -c "import acteval as ae; assert ae.rmse([1, 2], [1, 2]) == 0; - assert ae.__version__ == '0.3.0'" + assert ae.__version__ == '1.0.0'" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8217dc5..00fc96b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 with: - python-version: "3.13" + python-version: "3.14" cache: pip - run: python -m pip install --upgrade pip build - run: python -m build diff --git a/CHANGELOG.md b/CHANGELOG.md index 36ea30e..9506e22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## 1.0.0 - Unreleased + +### Stable API and packaging + +- Declared the documented public API stable under Semantic Versioning. +- Centralized package version metadata and moved to the Production/Stable + classifier. +- Added a formal API stability policy and security reporting policy. +- Reworked the README as a complete Python-library landing page. + +### v0.5 reporting and monitoring layer + +- Added portfolio-segment evaluation and aligned segment model comparison. +- Added chronological evaluation with signed changes from a baseline period. +- Added weighted reference-quantile prediction drift and PSI contributions. +- Added dependency-light standalone HTML reports and CSV/JSON/HTML exports. +- Added a supported helper for exporting Matplotlib figures. + +### v0.4 statistical inference layer + +- Added reproducible percentile-bootstrap intervals for evaluation metrics. +- Added objective-aware paired bootstrap comparisons against named references. +- Added fixed-bin stratified bootstrap intervals for calibration tables. +- Added structured inference results and CSV export. + ## 0.3.0 - 2026-08-12 - Added generic benchmarked financial decision regret. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..dfa8b56 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,13 @@ +cff-version: 1.2.0 +message: "If you use ActEval in research, please cite this software." +title: "ActEval: model-agnostic evaluation of actuarial predictive models" +type: software +authors: + - family-names: Manai + given-names: Amine + email: amine.manai@esprit.tn +repository-code: "https://github.com/aminemanai2003/acteval" +url: "https://pypi.org/project/acteval-insurance/" +license: Apache-2.0 +version: 1.0.0 +date-released: 2026-08-12 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d6034a6..a72384f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,11 @@ pytest python -m build ``` +CI must pass on every Python version listed in `pyproject.toml`. Public API +changes must update the API guide, README, changelog, tests, and stability +policy where relevant. Backward-incompatible changes require a major release +unless they correct a security issue or materially invalid calculation. + New metrics must document their definition, direction or target, supported tasks, domain assumptions, limitations, and references where applicable. Add basic, weighted, edge, invalid-input, and known-value tests. Do not introduce a diff --git a/README.md b/README.md index 3f0c756..272979f 100644 --- a/README.md +++ b/README.md @@ -1,211 +1,245 @@ # ActEval -Evaluate actuarial predictive models beyond predictive accuracy. - -ActEval is a model-agnostic Python framework for non-life insurance model -evaluation. It accepts prediction arrays instead of fitted model objects, so it -works after GLMs, scikit-learn pipelines, XGBoost, CatBoost, or neural networks. - -ActEval reports accuracy, calibration, discrimination, and observed-tail -behavior separately. It does not create an arbitrary overall score or claim -that one model is universally best. - -Version 0.2 also evaluates full predictive distributions using proper scoring -rules and keeps uncertainty diagnostics separate from model-quality claims. -Version 0.3 adds explicit, benchmarked financial decision diagnostics without -collapsing pricing, reserving, capital, and reinsurance into one score. +[![PyPI version](https://img.shields.io/pypi/v/acteval-insurance.svg)](https://pypi.org/project/acteval-insurance/) +[![Python versions](https://img.shields.io/pypi/pyversions/acteval-insurance.svg)](https://pypi.org/project/acteval-insurance/) +[![CI](https://github.com/aminemanai2003/acteval/actions/workflows/ci.yml/badge.svg)](https://github.com/aminemanai2003/acteval/actions/workflows/ci.yml) +[![License](https://img.shields.io/pypi/l/acteval-insurance.svg)](https://github.com/aminemanai2003/acteval/blob/main/LICENSE) +[![Typed](https://img.shields.io/badge/typing-typed-blue.svg)](https://peps.python.org/pep-0561/) + +**Model-agnostic evaluation for actuarial predictive models.** + +ActEval evaluates prediction arrays—not fitted model objects—across accuracy, +calibration, discrimination, probabilistic quality, uncertainty, observed-tail +risk, and financial decisions. It works with outputs from GLMs, scikit-learn, +XGBoost, CatBoost, neural networks, or any other modelling stack. + +The project is designed for non-life insurance pricing workflows. It keeps +actuarial objectives separate and never creates an arbitrary universal model +score. + +## Why ActEval? + +A model with lower RMSE can still have worse aggregate calibration, weaker +large-loss behavior, or a less favorable pricing consequence. ActEval makes +those trade-offs visible through explicit metrics and reproducible metadata. + +| Capability | Included diagnostics | +|---|---| +| Point predictions | MAE, RMSE, Poisson/Gamma/Tweedie deviance | +| Calibration | A/E, calibration by risk quantile, weighted calibration error | +| Discrimination | Gini, normalized Gini, lift | +| Tail risk | Observed-tail MAE, RMSE, A/E, large-loss bias | +| Predictive distributions | CRPS, log, Brier, quantile, and interval scores | +| Uncertainty | Coverage, width, variance, entropy, bootstrap intervals | +| Model comparison | Metric-specific ranking and paired bootstrap differences | +| Monitoring | Segment reports, temporal validation, prediction drift/PSI | +| Decisions | Pricing regret, loss ratio, reserve/capital shortfall, reinsurance | +| Reporting | DataFrame, dictionary, CSV, JSON, HTML, and plot export | ## Installation -Install the package from PyPI: +ActEval requires Python 3.11 or newer. ```bash python -m pip install acteval-insurance ``` -For plots: +Install the optional plotting support with: ```bash python -m pip install "acteval-insurance[plot]" ``` -The distribution is named `acteval-insurance` because `acteval` is occupied by -an unrelated project on PyPI. The import name remains `acteval`. +The distribution name is `acteval-insurance` because `acteval` was already +occupied on PyPI. The import remains concise: + +```python +import acteval as ae +``` ## Quick start +ActEval accepts ordinary NumPy-compatible arrays and returns structured result +objects. + ```python import acteval as ae +y_true = [0.0, 0.4, 1.0, 2.0, 4.0, 7.0] +y_pred = [0.1, 0.5, 0.9, 1.8, 3.6, 6.4] +exposure = [1.0, 0.5, 1.2, 0.8, 1.5, 2.0] + result = ae.evaluate( - y_true=[0.0, 0.4, 1.0, 2.0, 4.0, 7.0], - y_pred=[0.1, 0.5, 0.9, 1.8, 3.6, 6.4], - exposure=[1.0, 0.5, 1.2, 0.8, 1.5, 2.0], + y_true, + y_pred, + exposure=exposure, task="claim_frequency", - metrics=["rmse", "poisson_deviance", "ae_ratio", "normalized_gini"], ) -print(result.summary()) +print(result.to_dataframe()) ``` -Task defaults provide a broader report, including 95% observed-tail metrics. +Task defaults provide a balanced report. Metrics can be selected explicitly: -## Model comparison +```python +result = ae.evaluate( + y_true, + y_pred, + task="claim_frequency", + metrics=[ + "rmse", + "poisson_deviance", + "ae_ratio", + "normalized_gini", + "tail_ae_95", + ], +) +``` + +Parameterized metrics use `MetricSpec`, keeping every assumption in result +metadata: + +```python +result = ae.evaluate( + y_true, + y_pred, + task="pure_premium", + metrics=[ + ae.MetricSpec("tweedie_deviance", {"power": 1.7}), + ae.MetricSpec("tail_mae", {"quantile": 0.99}, label="tail_mae_99"), + ], +) +``` + +## Compare models ```python comparison = ae.compare( - y_true=y, - predictions={ + y_true, + { "GLM": glm_predictions, - "CatBoost": catboost_predictions, - "XGBoost": xgb_predictions, + "Gradient boosting": boosting_predictions, }, exposure=exposure, task="claim_frequency", ) print(comparison.to_dataframe()) -print(comparison.rank(metric="poisson_deviance")) +print(comparison.rank("poisson_deviance")) ``` -`rank()` uses a metric's documented direction. Target metrics such as A/E are -ranked by distance from 1. Rankings remain metric-specific. - -## Accuracy can disagree with tail calibration +Rankings are metric-specific. Target metrics such as A/E are ranked by distance +from their target; ActEval does not declare one model universally best. -The example below deliberately creates two models: +## Quantify sampling uncertainty -- Model A makes moderate errors on many ordinary risks but predicts large - observed outcomes accurately. -- Model B improves ordinary-risk predictions and overall RMSE while - underpredicting the observed tail. +Version 1.0 includes the inference layer introduced for the v0.4 roadmap. +Rows, predictions, exposures, and weights are resampled jointly. ```python -import numpy as np -import acteval as ae - -y = np.r_[np.tile([0.5, 1.0, 1.5, 1.0, 0.5], 19), np.repeat(10.0, 5)] -model_a = np.r_[y[:95] + 0.5, np.repeat(10.0, 5)] -model_b = np.r_[y[:95], np.repeat(9.0, 5)] - -tradeoff = ae.compare( - y, - {"Model A": model_a, "Model B": model_b}, +intervals = ae.bootstrap_evaluate( + y_true, + y_pred, + exposure=exposure, task="claim_frequency", - metrics=["rmse", "poisson_deviance", "tail_ae_95"], + metrics=["rmse", "ae_ratio", "normalized_gini", "tail_ae_95"], + n_resamples=2_000, + confidence_level=0.95, + random_state=42, ) -print(tradeoff.to_dataframe()) -``` - -Model B has lower overall RMSE and deviance, while Model A has tail A/E equal -to 1. The appropriate choice depends on the actuarial objective. - -## Input and exposure contract - -`y_true` and `y_pred` must be finite, one-dimensional, nonnegative arrays on -the same scale. -- For claim frequency, use frequency rates for both arrays and provide policy - exposure as `exposure`. -- For pure premium, use pure-premium rates for both arrays and provide exposure - when portfolio-volume weighting is desired. -- For severity, use claim severities. Exposure is optional and usually - unnecessary; claim-level `sample_weight` is normally more meaningful. -- If both are supplied, effective weight is `sample_weight * exposure`. - -ActEval does not silently convert raw claim counts into rates. - -## Parameterized metrics +print(intervals.to_dataframe()) +``` -Use `MetricSpec` whenever a parameter should be explicit and reproducible: +For model comparisons, paired resampling evaluates every model on the same +bootstrap rows. Negative `objective_delta` favors the candidate model after +accounting for whether a metric is minimized, maximized, or has a target. ```python -result = ae.evaluate( - y, - predictions, - task="pure_premium", - metrics=[ - ae.MetricSpec("tweedie_deviance", {"power": 1.7}), - ae.MetricSpec("tail_mae", {"quantile": 0.99}, label="tail_mae_99"), - ], +paired = ae.paired_bootstrap_compare( + y_true, + {"Current GLM": glm_predictions, "Candidate": boosting_predictions}, + reference="Current GLM", + task="claim_frequency", + metrics=["poisson_deviance", "ae_ratio", "normalized_gini"], + n_resamples=2_000, + random_state=42, ) ``` -Tail aliases such as `tail_mae_95`, `tail_rmse_99`, and `tail_ae_95` are also -accepted. Parameter values are retained in result metadata. +Confidence intervals are descriptive sampling-uncertainty estimates. Paired +comparisons are not automatically adjusted for multiple testing. + +## Segment and temporal monitoring -## Calibration, discrimination, and tail diagnostics +The v0.5 monitoring layer evaluates portfolio slices without changing the +meaning of the underlying metrics. ```python -calibration = ae.calibration_by_quantile(y, predictions, n_bins=10) -lift = ae.lift_by_quantile(y, predictions, n_bins=10) +segments = ae.evaluate_by_segment( + y_true, + y_pred, + segment_labels, + task="claim_frequency", + exposure=exposure, + metrics=["ae_ratio", "normalized_gini", "tail_ae_95"], +) -print(calibration.to_dataframe()) -print(lift.to_dataframe()) +timeline = ae.evaluate_over_time( + y_true, + y_pred, + accounting_period, + task="claim_frequency", + exposure=exposure, + metrics=["poisson_deviance", "ae_ratio"], +) -ae.plot_calibration(y, predictions) -ae.plot_lift(y, predictions) -ae.plot_residuals(y, predictions) -ae.plot_tail_diagnostics(y, predictions, quantile=0.95) +drift = ae.prediction_drift( + reference_predictions, + current_predictions, + n_bins=10, +) ``` +Prediction drift uses fixed, weighted reference-quantile bins and reports PSI +contributions. ActEval intentionally applies no universal PSI alert threshold. + ## Predictive distributions -Built-in vectorized adapters provide one predictive distribution per +Built-in vectorized adapters represent one predictive distribution per observation: -- `PoissonDistribution(mu)`; -- `NegativeBinomialDistribution(mean, dispersion)`; -- `GammaDistribution(mean, shape)`; -- `LognormalDistribution(meanlog, sdlog)`; -- `EmpiricalDistribution(samples)`; -- `TweedieDistribution(mean, power, dispersion)` for compound - Poisson-Gamma `1 < power < 2`. +- `PoissonDistribution(mu)` +- `NegativeBinomialDistribution(mean, dispersion)` +- `GammaDistribution(mean, shape)` +- `LognormalDistribution(meanlog, sdlog)` +- `TweedieDistribution(mean, power, dispersion)` for `1 < power < 2` +- `EmpiricalDistribution(samples)` for joint or independent scenario draws ```python -poisson = ae.PoissonDistribution(mu=poisson_means) -negative_binomial = ae.NegativeBinomialDistribution( - mean=nb_means, - dispersion=nb_dispersion, -) +poisson = ae.PoissonDistribution(mu=frequency_predictions) -distribution_comparison = ae.compare_distributions( - y_true=claim_counts, - distributions={ - "Poisson": poisson, - "Negative Binomial": negative_binomial, - }, - exposure=exposure, +distribution_result = ae.evaluate_distribution( + claim_counts, + poisson, task="claim_frequency", + exposure=exposure, metrics=[ - ae.MetricSpec("crps", {"n_samples": 5000, "random_state": 42}), + ae.MetricSpec("crps", {"n_samples": 5_000, "random_state": 42}), "log_score", - ae.MetricSpec("brier_score", {"threshold": 0}), - ae.MetricSpec("interval_score", {"coverage": 0.9}), + ae.MetricSpec("interval_score", {"coverage": 0.90}), ], ) - -print(distribution_comparison.to_dataframe()) ``` Samples have shape `(n_samples, n_observations)`. Scalar quantiles have shape `(n_observations,)`; vector quantiles have shape -`(n_quantiles, n_observations)`. CRPS randomness is explicitly seeded and -recorded in result metadata. - -Tweedie sampling uses the exact compound representation. CDF and log-density -evaluation use a numerical series implementation; quantiles use deterministic -Monte Carlo. Entropy is a seeded Monte Carlo estimate of `-E[log_prob(X)]` and -is only comparable under the same mixed distribution measure. Empirical draws -are treated as a discrete distribution: repeated values determine probability -mass, and unseen values have log probability `-inf`. +`(n_quantiles, n_observations)`. ## Decision-aware evaluation -Decision functions always expose their financial loss and benchmark. Regret is -`model financial loss - benchmark financial loss` in the loss function's unit. -It may be negative when the model decision outperforms the benchmark. Relative -regret is omitted when benchmark loss is zero. +Financial decisions always expose their loss function and named benchmark. +Regret is reported in the financial loss function's unit. ```python premiums = ae.premium_from_distribution( @@ -222,78 +256,53 @@ pricing = ae.pricing_regret( overpricing_cost=1.0, benchmark_name="current tariff", ) +``` -loss_ratio = ae.loss_ratio_impact( - realized_loss, - premiums, - target_loss_ratio=0.70, -) +ActEval also provides loss-ratio impact, reserve and capital shortfall, and +quoted stop-loss reinsurance selection. These are explicit decision models, +not interchangeable measures of predictive accuracy. -reserve = ae.reserve_shortfall(realized_loss, held_reserve) -capital = ae.capital_shortfall(realized_loss, available_capital) -``` +## Reports and exports -Stop-loss reinsurance selection compares quoted options under one explicit -rule: premium plus expected retained aggregate loss plus a user-selected cost -of VaR or expected-shortfall capital. +Result objects support DataFrames, dictionaries, printable summaries, and +standalone HTML reports: ```python -options = [ - ae.ReinsuranceOption("No cover", retention=1_000_000, premium=0), - ae.ReinsuranceOption("100k retention", retention=100_000, premium=25_000), -] - -selection = ae.select_reinsurance_option( - aggregate_loss_distribution, - options, - risk_measure="expected_shortfall", - risk_quantile=0.995, - capital_cost_rate=0.10, - random_state=42, -) +result.save_html("reports/frequency-evaluation.html") +comparison.save_html("reports/model-comparison.html") -realized = ae.reinsurance_decision_regret( - aggregate_loss=realized_annual_losses, - selected=selection.selected, - benchmark=options[0], -) +ae.export_table(comparison, "reports/model-comparison.csv") +ae.export_table(comparison, "reports/model-comparison.json") + +axis = ae.plot_calibration(y_true, y_pred, exposure=exposure) +ae.save_plot(axis, "reports/calibration.png", dpi=180) ``` -For reinsurance selection, each sampled row is a scenario and columns are -summed into portfolio aggregate loss. Dependence must therefore already be -represented by the supplied distribution's joint samples. Built-in parametric -adapters sample observation columns independently; use `EmpiricalDistribution` -with joint scenario draws when portfolio dependence matters. - -## Supported MVP metrics - -| Metric | Category | Interpretation | -|---|---|---| -| `mae` | accuracy | Lower is better | -| `rmse` | accuracy | Lower is better | -| `poisson_deviance` | accuracy | Lower; frequency only | -| `gamma_deviance` | accuracy | Lower; positive severity only | -| `tweedie_deviance` | accuracy | Lower; explicit power required | -| `ae_ratio` | calibration | Target is 1 | -| `weighted_calibration_error` | calibration | Lower is better | -| `gini` | discrimination | Higher is better | -| `normalized_gini` | discrimination | Perfect ordering is 1 | -| `lift` | discrimination | Higher means stronger top-group concentration | -| `tail_mae` | tail risk | Lower is better | -| `tail_rmse` | tail risk | Lower is better | -| `tail_ae_ratio` | tail risk | Target is 1 | -| `crps` | probabilistic | Lower is better | -| `log_score` | probabilistic | Lower is better | -| `brier_score` | probabilistic | Lower is better for an explicit event | -| `quantile_score` | probabilistic | Lower is better | -| `interval_score` | probabilistic | Lower is better | -| `interval_coverage` | uncertainty | Compare with requested coverage | -| `interval_width` | uncertainty | Sharpness; no universal direction | -| `predictive_variance` | uncertainty | No universal direction | -| `predictive_entropy` | uncertainty | No universal direction | - -Use `ae.list_metrics()` for machine-readable registry metadata. Exact formulas -and limitations are in [the metric reference](docs/metric-reference.md). +HTML reports contain no JavaScript or remote assets and can be archived for +offline review. + +## Input contract + +- `y_true` and `y_pred` are finite, one-dimensional, nonnegative arrays on the + same scale. +- Frequency and pure-premium rates should be supplied with policy exposure. +- Severity observations are normally claim-level; `sample_weight` is often + more meaningful than exposure. +- When both are present, effective weight is `sample_weight * exposure`. +- ActEval does not silently convert claim counts to rates. +- Observed-tail diagnostics select rows using realized outcomes and are + retrospective—not predictive tail probabilities. + +## Documentation + +- [API guide](https://github.com/aminemanai2003/acteval/blob/main/docs/api.md) +- [Metric reference](https://github.com/aminemanai2003/acteval/blob/main/docs/metric-reference.md) +- [Bootstrap inference](https://github.com/aminemanai2003/acteval/blob/main/docs/inference.md) +- [Monitoring](https://github.com/aminemanai2003/acteval/blob/main/docs/monitoring.md) +- [Reporting](https://github.com/aminemanai2003/acteval/blob/main/docs/reporting.md) +- [Decision reference](https://github.com/aminemanai2003/acteval/blob/main/docs/decision-reference.md) +- [API stability policy](https://github.com/aminemanai2003/acteval/blob/main/docs/stability.md) +- [Migrating from 0.3 to 1.0](https://github.com/aminemanai2003/acteval/blob/main/docs/migration-1.0.md) ## Development @@ -303,26 +312,19 @@ cd acteval python -m venv .venv python -m pip install -e ".[dev]" ruff check . +ruff format --check . mypy src/acteval pytest python -m build ``` -See [CONTRIBUTING.md](CONTRIBUTING.md) and the -[implementation audit](docs/plan-audit.md). - -## Implemented releases - -- v0.1: point-prediction accuracy, calibration, discrimination, tail - diagnostics, comparisons, and plotting. -- v0.2: predictive-distribution scores and uncertainty diagnostics. -- v0.3: explicit benchmarked pricing, loss-ratio, reserve, capital, and - reinsurance financial consequences. +Contributions are welcome. Read [CONTRIBUTING.md](https://github.com/aminemanai2003/acteval/blob/main/CONTRIBUTING.md) +and the [security policy](https://github.com/aminemanai2003/acteval/blob/main/SECURITY.md) +before opening a pull request or reporting a vulnerability. -The original v0.1-v0.3 implementation plan is complete. Remaining work is -release operations and future scope, not missing behavior from that plan. See -the [completion audit](docs/plan-audit.md) for boundaries and evidence. +## Versioning and license -## License +ActEval follows Semantic Versioning from 1.0 onward. Public compatibility and +deprecation guarantees are documented in the stability policy. -Apache-2.0. +Licensed under the [Apache License 2.0](https://github.com/aminemanai2003/acteval/blob/main/LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..e4ab108 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,15 @@ +# Security policy + +## Supported versions + +Security fixes are provided for the latest ActEval release. + +## Reporting a vulnerability + +Do not open a public issue for a suspected vulnerability. Use GitHub's private +vulnerability reporting feature under the repository's **Security** tab, or +contact the maintainer at `amine.manai@esprit.tn` when private reporting is not +available. + +Include the affected version, reproduction steps, impact, and any suggested +mitigation. Please allow reasonable time for investigation before disclosure. diff --git a/docs/api.md b/docs/api.md index b1bbb64..fedeb13 100644 --- a/docs/api.md +++ b/docs/api.md @@ -93,3 +93,34 @@ absolute regret, optional relative regret, and benchmark/loss metadata. `decision_regret()` is the low-level extension point. Callers provide a loss function returning finite nonnegative financial loss per observation. Regret can be negative and is never interpreted across different loss functions. + +## Bootstrap inference + +- `bootstrap_evaluate()` returns point estimates and percentile confidence + intervals for selected metrics. +- `paired_bootstrap_compare()` returns objective-aware paired differences + against a named reference model. Negative differences favor the candidate. +- `bootstrap_calibration_by_quantile()` returns fixed-bin intervals for mean + prediction, mean observation, and A/E. + +All row-level arrays are resampled jointly. Seeds, attempts, rejected resamples, +confidence levels, and sample counts are retained in metadata. See +[`inference.md`](inference.md) for interpretation and limitations. + +## Segment and temporal monitoring + +- `evaluate_by_segment()` evaluates one model within caller-defined groups. +- `compare_by_segment()` compares aligned models in every retained group. +- `evaluate_over_time()` sorts period labels and reports changes from baseline. +- `prediction_drift()` reports weighted reference-bin PSI and mean shifts. + +Monitoring is descriptive and does not apply universal materiality thresholds. +See [`monitoring.md`](monitoring.md). + +## Reporting and exports + +`render_html_report()` and `save_html_report()` generate standalone documents. +`export_table()` writes CSV, JSON, or HTML, and `save_plot()` exports the figure +owned by a Matplotlib axis. `EvaluationResult` and `ComparisonResult` expose +`to_html()` and `save_html()` convenience methods. See +[`reporting.md`](reporting.md). diff --git a/docs/inference.md b/docs/inference.md new file mode 100644 index 0000000..e8c11b0 --- /dev/null +++ b/docs/inference.md @@ -0,0 +1,58 @@ +# Bootstrap inference + +ActEval separates point estimates from sampling uncertainty. The inference API +uses nonparametric percentile bootstrap intervals and records its sample count, +confidence level, seed, retry count, and method. + +## Single-model intervals + +`bootstrap_evaluate()` resamples complete rows. Observations, predictions, +exposures, and sample weights therefore remain aligned. It returns a +`BootstrapEvaluationResult` containing the ordinary `EvaluationResult` plus one +`ConfidenceInterval` per metric. + +Resamples where an estimand is undefined—for example, a normalized Gini with a +constant outcome—are rejected and retried. Failure to obtain the requested +number of valid resamples raises `InputValidationError`; it is never silently +reported with a smaller sample count. + +The default is 1,000 resamples. At least 100 are required. For final reporting, +2,000 or more is generally preferable when computation permits. + +## Paired comparisons + +`paired_bootstrap_compare()` uses identical resampled rows for every model. +Each result includes: + +- the candidate and reference point estimates; +- their raw metric difference; +- an objective-aware difference and percentile interval; +- the bootstrap standard error; +- whether the interval excludes zero. + +Objective differences always use a lower-is-better convention: + +- minimized metric: `candidate - reference`; +- maximized metric: `reference - candidate`; +- target metric: candidate absolute target distance minus reference distance. + +Negative values therefore favor the candidate under that metric's objective. +This transformation does not combine metrics or claim universal superiority. + +Intervals are not multiplicity-adjusted and should not be interpreted as a +substitute for a pre-specified model-governance decision rule. + +## Calibration intervals + +`bootstrap_calibration_by_quantile()` fixes risk bins from the original +predictions and resamples within each populated bin. This preserves each bin's +interpretation while estimating uncertainty for mean prediction, mean +observation, and A/E. Prediction ties remain together and zero-effective-weight +rows do not enter the bootstrap population. + +## Reproducibility + +Supply an integer `random_state` in governed or published work. `None` requests +non-deterministic sampling. Bootstrap output describes sampling variability of +the supplied evaluation portfolio; it does not account automatically for model +fitting uncertainty, temporal dependence, clustering, or data leakage. diff --git a/docs/migration-1.0.md b/docs/migration-1.0.md new file mode 100644 index 0000000..87ab037 --- /dev/null +++ b/docs/migration-1.0.md @@ -0,0 +1,39 @@ +# Migrating from 0.3 to 1.0 + +ActEval 1.0 is backward compatible with the documented 0.3 public API. Existing +point-prediction, predictive-distribution, plotting, and decision-aware calls do +not require changes. + +## Package and import names + +The installation and import names are unchanged: + +```bash +python -m pip install --upgrade acteval-insurance +``` + +```python +import acteval as ae +``` + +## New capabilities + +- Use `bootstrap_evaluate()` to add confidence intervals to existing metric + selections. +- Use `paired_bootstrap_compare()` when a named candidate must be compared with + a reference on aligned resamples. +- Use `evaluate_by_segment()`, `compare_by_segment()`, and + `evaluate_over_time()` for portfolio monitoring. +- Use `prediction_drift()` for fixed-reference score-distribution diagnostics. +- Use result `save_html()` methods or `export_table()` for governed artifacts. + +## Compatibility notes + +- `__version__` remains available from `acteval` and now has a single packaging + source of truth. +- The documented names in `acteval.__all__` are covered by the 1.x stability + policy. +- `summary()` remains intended for people. Use `to_dict()` or `to_dataframe()` + for integrations. +- No composite score, automatic PSI threshold, or universal model ranking was + introduced. diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 0000000..04695de --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,40 @@ +# Segment and temporal monitoring + +Monitoring functions reuse the same validated evaluation pipeline as portfolio +reports. They do not redefine metrics or create a monitoring score. + +## Portfolio segments + +`evaluate_by_segment()` returns an `EvaluationResult` for every segment meeting +`min_observations`. `compare_by_segment()` repeats an aligned multi-model +comparison within each segment. Missing or empty labels are rejected; labels +that would collide after string normalization are also rejected. + +Small segments are recorded in `skipped_segments`. They are not silently folded +into another group. Statistical credibility remains the caller's responsibility. + +## Temporal validation + +`evaluate_over_time()` treats labels as ordered strings, sorts them, and reports +each metric's signed change from the first retained period. ISO dates and +lexically ordered labels such as `2026-Q1` satisfy the ordering contract. + +Changes are descriptive. A positive change may be favorable, unfavorable, or +neither depending on the metric's direction or target. + +## Prediction drift + +`prediction_drift()` constructs fixed cut points from weighted reference-score +quantiles. It reports: + +- population stability index (PSI); +- each bin's reference/current portfolio share and PSI contribution; +- absolute and relative mean-prediction shifts. + +Zero bin shares are clipped only inside the logarithm calculation using the +recorded `epsilon`. Reported portfolio shares remain unmodified. + +PSI has no universal significance or action threshold. ActEval deliberately +does not label a value as acceptable, moderate, or severe. Operational limits +should be calibrated to the portfolio, score behavior, governance policy, and +cost of action. diff --git a/docs/plan-audit.md b/docs/plan-audit.md index b36ffc6..83134fc 100644 --- a/docs/plan-audit.md +++ b/docs/plan-audit.md @@ -100,3 +100,27 @@ universal model score. - The distribution is ready for PyPI trusted publishing as `acteval-insurance`, but registering the publisher and issuing the first release require repository/PyPI owner actions. + +## Version 1.0 extension audit + +The v0.4, v0.5, and v1.0 extension roadmap is implemented as the 1.0 public +surface: + +| Roadmap requirement | Evidence | +|---|---| +| Bootstrap confidence intervals | `bootstrap_evaluate()` and structured confidence intervals | +| A/E, Gini, lift, and tail uncertainty | Registry-driven inference supports every point metric selected by the caller | +| Paired model comparison | Shared row resamples and objective-aware candidate/reference deltas | +| Calibration uncertainty | Fixed risk bins with stratified intervals for prediction, observation, and A/E | +| HTML reports | Standalone escaped documents with no remote assets or JavaScript | +| Exportable tables and plots | CSV/JSON/HTML `export_table()` and `save_plot()` | +| Portfolio segments | Single- and multi-model segment result objects | +| Temporal validation | Ordered period metrics and signed baseline changes | +| Prediction drift | Weighted reference-quantile PSI contributions and mean shifts | +| Stable API | Semantic Versioning and documented compatibility/deprecation policy | +| Performance | Cached metric signature validation for repeated bootstrap/segment evaluation | +| Documentation | Dedicated inference, monitoring, reporting, migration, stability, security, and README audit documents | +| Production metadata | Single-source 1.0 version, stable classifier, typed marker, citation metadata | + +Completion requires local and remote formatting, lint, strict typing, full tests, +source/wheel builds, clean artifact installation, and public API smoke tests. diff --git a/docs/readme-audit.md b/docs/readme-audit.md new file mode 100644 index 0000000..47f988f --- /dev/null +++ b/docs/readme-audit.md @@ -0,0 +1,32 @@ +# README audit for 1.0 + +The pre-1.0 README accurately described the point, probabilistic, and decision +APIs, but it had become a chronological implementation narrative rather than a +library landing page. The 1.0 review found these gaps: + +| Area | Earlier state | 1.0 resolution | +|---|---|---| +| Project signals | No badges or supported-version summary | Added PyPI, Python, CI, license, and typing badges | +| Value proposition | Correct but spread across version paragraphs | Led with model-agnostic purpose and actuarial trade-offs | +| Feature discovery | Long sequential prose | Added a capability-to-diagnostics map | +| Quick start | Point evaluation only | Kept a minimal runnable example and explicit metric selection | +| Inference | Not present | Added interval and paired-comparison workflows with interpretation | +| Monitoring | Not present | Added segment, temporal, and drift examples | +| Reporting | Plot calls only | Added HTML, CSV, JSON, and plot export examples | +| Contracts | Exposure contract was present | Consolidated scale, weighting, and observed-tail limitations | +| Navigation | Three documentation links | Added focused API, metric, inference, monitoring, reporting, decision, and stability links | +| Maintenance | Development commands only | Added contribution, security, versioning, and license paths | +| Terminology | “Supported MVP metrics” remained after v0.3 | Replaced with a version-neutral capability overview | + +## Verification criteria + +- Installation uses the published `acteval-insurance` distribution and + `import acteval` module. +- Every shown public function is exported from `acteval.__all__`. +- Runnable examples use defined variables or clearly named placeholders in + model-comparison contexts. +- Scientific limitations appear next to the feature they constrain. +- Relative documentation links use absolute GitHub targets so they work on + both GitHub and PyPI. +- Development commands match CI. +- The README makes no universal model-quality or PSI-threshold claim. diff --git a/docs/reporting.md b/docs/reporting.md new file mode 100644 index 0000000..e8bed53 --- /dev/null +++ b/docs/reporting.md @@ -0,0 +1,31 @@ +# Reporting and export + +ActEval results are data objects first. Reporting functions do not recompute or +reinterpret metrics. + +## Structured output + +Core and extended result objects provide `to_dict()`, `to_dataframe()`, and +`summary()`. `export_table()` writes: + +- CSV for the primary table; +- JSON for the full structured result; +- standalone HTML for human review. + +The output format defaults to the file extension and can be supplied explicitly. + +## HTML reports + +`render_html_report()` returns a complete UTF-8 document. `save_html_report()` +writes it to disk. `EvaluationResult` and `ComparisonResult` also expose +convenience `to_html()` and `save_html()` methods. + +Reports contain inline CSS, no JavaScript, and no remote assets. Titles, labels, +and serialized metadata are escaped. The report includes a reminder that +metrics represent different actuarial objectives. + +## Plots + +Plotting functions return a Matplotlib axis. `save_plot()` saves the owning +figure with an explicit DPI and tight bounding box. Plot calculation remains an +optional dependency and is separate from metric computation. diff --git a/docs/stability.md b/docs/stability.md new file mode 100644 index 0000000..b83dbce --- /dev/null +++ b/docs/stability.md @@ -0,0 +1,34 @@ +# API stability policy + +ActEval follows Semantic Versioning beginning with version 1.0.0. + +## Public API + +Names documented in the README, files under `docs/`, and `acteval.__all__` are +public. Their call signatures, return types, field meanings, and documented +mathematical conventions are compatibility commitments within a major release. + +Modules, functions, and attributes beginning with an underscore are internal. +Registry implementation details and exact text formatting of `summary()` are not +stable serialization formats; use `to_dict()` or `to_dataframe()` instead. + +## Deprecations + +An incompatible public change will normally be introduced with a documented +deprecation warning and retained for at least one minor release before removal. +Immediate removal may occur for a security issue, materially incorrect +calculation, or behavior that cannot be preserved safely. Such changes will be +called out prominently in the changelog. + +## Numerical compatibility + +Metric definitions, weighting direction, tail-selection rules, distribution +parameterizations, and decision-loss conventions are public behavior. Floating +point results can vary slightly across supported NumPy/SciPy versions and +platforms. Seeded Monte Carlo and bootstrap methods promise reproducible random +streams for a fixed supported dependency stack, not bitwise equality forever. + +## Supported Python + +The package metadata and CI matrix define supported Python versions. Dropping a +Python version requires at least a minor release and changelog entry. diff --git a/examples/inference_and_monitoring.py b/examples/inference_and_monitoring.py new file mode 100644 index 0000000..b7f7ddb --- /dev/null +++ b/examples/inference_and_monitoring.py @@ -0,0 +1,67 @@ +"""End-to-end inference, monitoring, and reporting example.""" + +from pathlib import Path + +import numpy as np + +import acteval as ae + +rng = np.random.default_rng(2026) +n_policies = 240 +exposure = rng.uniform(0.4, 1.0, n_policies) +risk_score = rng.lognormal(-1.5, 0.55, n_policies) +y_true = rng.poisson(risk_score * exposure) / exposure +current = risk_score * 1.08 +candidate = risk_score * 0.98 +segments = np.where(np.arange(n_policies) % 2, "Personal", "Commercial") +periods = np.repeat(["2026-Q1", "2026-Q2", "2026-Q3"], n_policies // 3) + +intervals = ae.bootstrap_evaluate( + y_true, + candidate, + task="claim_frequency", + exposure=exposure, + metrics=["poisson_deviance", "ae_ratio", "normalized_gini"], + n_resamples=200, + random_state=42, +) + +paired = ae.paired_bootstrap_compare( + y_true, + {"Current": current, "Candidate": candidate}, + task="claim_frequency", + reference="Current", + exposure=exposure, + metrics=["poisson_deviance", "ae_ratio"], + n_resamples=200, + random_state=42, +) + +segment_report = ae.evaluate_by_segment( + y_true, + candidate, + segments, + task="claim_frequency", + exposure=exposure, + metrics=["poisson_deviance", "ae_ratio"], +) + +timeline = ae.evaluate_over_time( + y_true, + candidate, + periods, + task="claim_frequency", + exposure=exposure, + metrics=["poisson_deviance", "ae_ratio"], +) + +drift = ae.prediction_drift(current[:80], candidate[-80:], n_bins=5) + +output = Path("acteval-example-report.html") +ae.save_html_report(paired, output, title="ActEval candidate review") + +print(intervals.to_dataframe()) +print(segment_report.to_dataframe()) +print(timeline.to_dataframe()) +print(drift.summary()) +print(f"Saved {output}") diff --git a/pyproject.toml b/pyproject.toml index a5df715..e724272 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "acteval-insurance" -version = "0.3.0" +dynamic = ["version"] description = "Model-agnostic evaluation of actuarial predictive models" readme = "README.md" requires-python = ">=3.11" @@ -15,13 +15,14 @@ authors = [ ] keywords = ["actuarial", "insurance", "model-evaluation", "calibration"] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Financial and Insurance Industry", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Typing :: Typed", ] dependencies = [ @@ -51,10 +52,15 @@ dev = [ Homepage = "https://github.com/aminemanai2003/acteval" Repository = "https://github.com/aminemanai2003/acteval" Issues = "https://github.com/aminemanai2003/acteval/issues" +Documentation = "https://github.com/aminemanai2003/acteval/tree/main/docs" +Changelog = "https://github.com/aminemanai2003/acteval/blob/main/CHANGELOG.md" [tool.hatch.build.targets.wheel] packages = ["src/acteval"] +[tool.hatch.version] +path = "src/acteval/_version.py" + [tool.pytest.ini_options] addopts = "--strict-config --strict-markers --cov=acteval --cov-report=term-missing --cov-fail-under=90" testpaths = ["tests"] diff --git a/src/acteval/__init__.py b/src/acteval/__init__.py index 6032ccc..5ab087a 100644 --- a/src/acteval/__init__.py +++ b/src/acteval/__init__.py @@ -1,5 +1,6 @@ """ActEval: evaluation tools for actuarial predictive models.""" +from acteval._version import __version__ from acteval.api import ( compare, compare_distributions, @@ -31,6 +32,18 @@ PoissonDistribution, TweedieDistribution, ) +from acteval.inference import ( + BootstrapEvaluationResult, + CalibrationIntervalBin, + CalibrationIntervalTable, + ConfidenceInterval, + PairedComparisonResult, + PairedMetricComparison, + bootstrap_calibration_by_quantile, + bootstrap_evaluate, + paired_bootstrap_compare, + save_interval_csv, +) from acteval.metrics import ( ae_ratio, brier_score, @@ -64,6 +77,17 @@ tweedie_deviance, weighted_calibration_error, ) +from acteval.monitoring import ( + DriftBin, + PredictionDriftResult, + SegmentComparisonResult, + SegmentEvaluationResult, + TemporalEvaluationResult, + compare_by_segment, + evaluate_by_segment, + evaluate_over_time, + prediction_drift, +) from acteval.plotting import ( plot_calibration, plot_lift, @@ -71,6 +95,12 @@ plot_tail_diagnostics, ) from acteval.registry import MetricDefinition, get_metric, list_metrics +from acteval.reporting import ( + export_table, + render_html_report, + save_html_report, + save_plot, +) from acteval.reports import ( CalibrationTable, ComparisonResult, @@ -81,9 +111,14 @@ from acteval.types import MetricSpec, PredictiveDistribution, Task __all__ = [ + "BootstrapEvaluationResult", + "CalibrationIntervalBin", + "CalibrationIntervalTable", "CalibrationTable", "ComparisonResult", + "ConfidenceInterval", "DecisionEvaluation", + "DriftBin", "EmpiricalDistribution", "EvaluationResult", "GammaDistribution", @@ -93,21 +128,31 @@ "MetricDefinition", "MetricSpec", "NegativeBinomialDistribution", + "PairedComparisonResult", + "PairedMetricComparison", "PoissonDistribution", + "PredictionDriftResult", "PredictiveDistribution", "ReinsuranceOption", "ReinsuranceSelection", + "SegmentComparisonResult", + "SegmentEvaluationResult", "ShortfallResult", "Task", "TaskDefinition", + "TemporalEvaluationResult", "TweedieDistribution", + "__version__", "ae_ratio", "asymmetric_absolute_loss", + "bootstrap_calibration_by_quantile", + "bootstrap_evaluate", "brier_score", "calibration_by_quantile", "capital_shortfall", "central_prediction_interval", "compare", + "compare_by_segment", "compare_distributions", "crps", "decision_regret", @@ -116,7 +161,10 @@ "distribution_interval_width", "distribution_quantile_score", "evaluate", + "evaluate_by_segment", "evaluate_distribution", + "evaluate_over_time", + "export_table", "gamma_deviance", "get_metric", "get_task_definition", @@ -131,11 +179,13 @@ "mae", "mean_interval_width", "normalized_gini", + "paired_bootstrap_compare", "plot_calibration", "plot_lift", "plot_residuals", "plot_tail_diagnostics", "poisson_deviance", + "prediction_drift", "prediction_interval_coverage", "predictive_entropy", "predictive_variance", @@ -144,9 +194,13 @@ "quantile_decision", "quantile_score", "reinsurance_decision_regret", + "render_html_report", "reserve_shortfall", "risk_group_lift", "rmse", + "save_html_report", + "save_interval_csv", + "save_plot", "select_reinsurance_option", "tail_ae_ratio", "tail_mae", @@ -154,5 +208,3 @@ "tweedie_deviance", "weighted_calibration_error", ] - -__version__ = "0.3.0" diff --git a/src/acteval/_version.py b/src/acteval/_version.py new file mode 100644 index 0000000..6ed46b7 --- /dev/null +++ b/src/acteval/_version.py @@ -0,0 +1,3 @@ +"""Single source of truth for the ActEval package version.""" + +__version__ = "1.0.0" diff --git a/src/acteval/api.py b/src/acteval/api.py index 8d9c0b1..011527f 100644 --- a/src/acteval/api.py +++ b/src/acteval/api.py @@ -2,7 +2,8 @@ import re from collections.abc import Mapping, Sequence -from inspect import signature +from functools import cache +from inspect import Signature, signature from typing import Any from numpy.typing import ArrayLike @@ -24,6 +25,13 @@ ) +@cache +def _metric_signature(function: Any) -> Signature: + """Cache call signatures used repeatedly by bootstrap and segment runs.""" + + return signature(function) + + def _parse_metric(selection: MetricSelection) -> MetricSpec: if isinstance(selection, MetricSpec): return selection @@ -80,7 +88,7 @@ def _evaluate_metric( **specification.parameters, } try: - signature(definition.function).bind( + _metric_signature(definition.function).bind( y_true, y_pred, **keyword_arguments, @@ -235,7 +243,7 @@ def _evaluate_distribution_metric( **specification.parameters, } try: - signature(definition.function).bind( + _metric_signature(definition.function).bind( y_true, distribution, **keyword_arguments, @@ -274,9 +282,9 @@ def evaluate_distribution( """Evaluate one predictive distribution per observation. The default report contains sample-approximated CRPS, predictive variance, - and 90% central-interval coverage and width. Log score and entropy are - opt-in because not every distribution adapter supplies validated density - or entropy methods. + and 90% central-interval coverage and width. Log score and entropy remain + opt-in because their interpretation depends on the distribution's base + measure and they do not share a universal optimization objective. """ resolved_task = validate_task(task) validated = validate_inputs( diff --git a/src/acteval/inference.py b/src/acteval/inference.py new file mode 100644 index 0000000..65644b4 --- /dev/null +++ b/src/acteval/inference.py @@ -0,0 +1,605 @@ +"""Bootstrap uncertainty for actuarial evaluation and model comparison.""" + +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Literal + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +from acteval.api import MetricSelection, compare, evaluate +from acteval.exceptions import ActEvalError, InputValidationError +from acteval.metrics import calibration_by_quantile +from acteval.reports import EvaluationResult +from acteval.types import NumericArray, Task +from acteval.utils import risk_bin_indices +from acteval.validation import combine_weights, validate_inputs, validate_probability + +BootstrapMethod = Literal["percentile"] + + +@dataclass(frozen=True, slots=True) +class ConfidenceInterval: + """A two-sided bootstrap confidence interval for one scalar estimand.""" + + estimate: float + lower: float + upper: float + standard_error: float + confidence_level: float + n_resamples: int + method: BootstrapMethod = "percentile" + + def to_dict(self) -> dict[str, float | int | str]: + """Return a JSON-friendly representation.""" + + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class BootstrapEvaluationResult: + """Point estimates and bootstrap intervals for one model.""" + + task: Task + point_estimate: EvaluationResult + intervals: dict[str, ConfidenceInterval] + metadata: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-friendly representation.""" + + return { + "task": self.task, + "point_estimate": self.point_estimate.to_dict(), + "intervals": { + metric: interval.to_dict() + for metric, interval in self.intervals.items() + }, + "metadata": dict(self.metadata), + } + + def to_dataframe(self) -> Any: + """Return one row per metric as a pandas DataFrame.""" + + import pandas as pd + + rows = [ + {"metric": metric, **interval.to_dict()} + for metric, interval in self.intervals.items() + ] + return pd.DataFrame.from_records(rows).set_index("metric") + + def summary(self) -> str: + """Return a concise printable interval table.""" + + return str(self.to_dataframe().to_string()) + + +@dataclass(frozen=True, slots=True) +class PairedMetricComparison: + """Paired bootstrap comparison of one model with a reference model. + + ``objective_delta`` is transformed so that negative values favor ``model``: + lower-is-better metrics use ``model - reference``; higher-is-better metrics + reverse the sign; target metrics compare absolute distance from the target. + """ + + model: str + reference: str + metric: str + model_estimate: float + reference_estimate: float + raw_delta: float + objective_delta: float + lower: float + upper: float + standard_error: float + confidence_level: float + n_resamples: int + + @property + def confidence_excludes_zero(self) -> bool: + """Whether the two-sided interval excludes no objective difference.""" + + return self.lower > 0 or self.upper < 0 + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-friendly representation.""" + + return { + **asdict(self), + "confidence_excludes_zero": self.confidence_excludes_zero, + } + + +@dataclass(frozen=True, slots=True) +class PairedComparisonResult: + """Metric-specific paired bootstrap comparisons against one reference.""" + + task: Task + reference: str + comparisons: tuple[PairedMetricComparison, ...] + metadata: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-friendly representation.""" + + return { + "task": self.task, + "reference": self.reference, + "comparisons": [comparison.to_dict() for comparison in self.comparisons], + "metadata": dict(self.metadata), + } + + def to_dataframe(self) -> Any: + """Return one row per model and metric as a pandas DataFrame.""" + + import pandas as pd + + frame = pd.DataFrame.from_records( + [comparison.to_dict() for comparison in self.comparisons] + ) + return frame.set_index(["model", "metric"]) + + def summary(self) -> str: + """Return a concise printable comparison table.""" + + return str(self.to_dataframe().to_string()) + + +@dataclass(frozen=True, slots=True) +class CalibrationIntervalBin: + """Fixed risk-bin calibration estimates with bootstrap intervals.""" + + bin: int + count: int + mean_prediction: ConfidenceInterval + mean_observed: ConfidenceInterval + ae_ratio: ConfidenceInterval + + def to_dict(self) -> dict[str, Any]: + """Flatten the bin into a tabular JSON-friendly record.""" + + record: dict[str, Any] = {"bin": self.bin, "count": self.count} + for name in ("mean_prediction", "mean_observed", "ae_ratio"): + interval = getattr(self, name) + record[name] = interval.estimate + record[f"{name}_lower"] = interval.lower + record[f"{name}_upper"] = interval.upper + return record + + +@dataclass(frozen=True, slots=True) +class CalibrationIntervalTable: + """Risk calibration table with fixed-bin stratified bootstrap intervals.""" + + bins: tuple[CalibrationIntervalBin, ...] + requested_bins: int + confidence_level: float + n_resamples: int + metadata: dict[str, Any] + + @property + def effective_bins(self) -> int: + """Number of populated bins after prediction ties are collapsed.""" + + return len(self.bins) + + def to_dict(self) -> list[dict[str, Any]]: + """Return one JSON-friendly record per bin.""" + + return [bin_result.to_dict() for bin_result in self.bins] + + def to_dataframe(self) -> Any: + """Return one row per risk bin as a pandas DataFrame.""" + + import pandas as pd + + return pd.DataFrame.from_records(self.to_dict()).set_index("bin") + + +def _validate_bootstrap(n_resamples: int, confidence_level: float) -> tuple[int, float]: + if isinstance(n_resamples, bool) or not isinstance(n_resamples, int): + raise InputValidationError("n_resamples must be an integer.") + if n_resamples < 100: + raise InputValidationError("n_resamples must be at least 100.") + return n_resamples, validate_probability(confidence_level, name="confidence_level") + + +def _interval( + estimate: float, + samples: NumericArray, + *, + confidence_level: float, +) -> ConfidenceInterval: + alpha = 1.0 - confidence_level + lower, upper = np.quantile(samples, [alpha / 2.0, 1.0 - alpha / 2.0]) + return ConfidenceInterval( + estimate=float(estimate), + lower=float(lower), + upper=float(upper), + standard_error=float(np.std(samples, ddof=1)), + confidence_level=confidence_level, + n_resamples=len(samples), + ) + + +def _slice_optional( + values: NumericArray | None, indices: NDArray[np.integer[Any]] +) -> NumericArray | None: + return None if values is None else values[indices.astype(np.intp)] + + +def bootstrap_evaluate( + y_true: ArrayLike, + y_pred: ArrayLike, + *, + task: str, + exposure: ArrayLike | None = None, + sample_weight: ArrayLike | None = None, + metrics: Sequence[MetricSelection] | None = None, + n_resamples: int = 1_000, + confidence_level: float = 0.95, + random_state: int | None = 0, +) -> BootstrapEvaluationResult: + """Estimate percentile bootstrap intervals for point-prediction metrics. + + Rows are resampled jointly, preserving alignment among observations, + predictions, exposures, and weights. Resamples where a requested estimand is + mathematically undefined are retried and the retry count is recorded. + """ + + count, confidence = _validate_bootstrap(n_resamples, confidence_level) + validated = validate_inputs( + y_true, + y_pred, + sample_weight=sample_weight, + exposure=exposure, + y_true_domain="nonnegative", + y_pred_domain="nonnegative", + ) + point = evaluate( + validated.y_true, + validated.y_pred, + task=task, + exposure=validated.exposure, + sample_weight=validated.sample_weight, + metrics=metrics, + ) + metric_samples: dict[str, list[float]] = {metric: [] for metric in point.metrics} + generator = np.random.default_rng(random_state) + attempts = 0 + max_attempts = max(count * 20, count + 100) + while len(next(iter(metric_samples.values()))) < count and attempts < max_attempts: + attempts += 1 + indices = generator.integers(0, len(validated.y_true), len(validated.y_true)) + try: + result = evaluate( + validated.y_true[indices], + validated.y_pred[indices], + task=task, + exposure=_slice_optional(validated.exposure, indices), + sample_weight=_slice_optional(validated.sample_weight, indices), + metrics=metrics, + ) + except ActEvalError: + continue + if not all(np.isfinite(value) for value in result.metrics.values()): + continue + for metric, value in result.metrics.items(): + metric_samples[metric].append(value) + completed = len(next(iter(metric_samples.values()))) + if completed < count: + raise InputValidationError( + f"Only {completed} valid bootstrap resamples could be generated after " + f"{attempts} attempts. Review degenerate outcomes or metric domains." + ) + intervals = { + metric: _interval( + point.metrics[metric], + np.asarray(samples, dtype=np.float64), + confidence_level=confidence, + ) + for metric, samples in metric_samples.items() + } + return BootstrapEvaluationResult( + point.task, + point, + intervals, + { + "method": "iid_percentile_bootstrap", + "n_resamples": count, + "confidence_level": confidence, + "random_state": random_state, + "attempts": attempts, + "failed_resamples": attempts - count, + }, + ) + + +def _objective_value(value: float, metadata: Mapping[str, Any]) -> float: + target = metadata.get("target") + if target is not None: + return abs(value - float(target)) + direction = metadata.get("higher_is_better") + if direction is True: + return -value + if direction is False: + return value + raise InputValidationError("Metric has no comparison direction or target.") + + +def paired_bootstrap_compare( + y_true: ArrayLike, + predictions: Mapping[str, ArrayLike], + *, + task: str, + reference: str | None = None, + exposure: ArrayLike | None = None, + sample_weight: ArrayLike | None = None, + metrics: Sequence[MetricSelection] | None = None, + n_resamples: int = 1_000, + confidence_level: float = 0.95, + random_state: int | None = 0, +) -> PairedComparisonResult: + """Compare models using shared row resamples and objective-aware deltas. + + Pairing removes bootstrap noise caused by evaluating models on different + resamples. A negative objective delta favors the candidate model. Confidence + intervals quantify sampling uncertainty and are not multiple-test adjusted. + """ + + count, confidence = _validate_bootstrap(n_resamples, confidence_level) + point = compare( + y_true, + predictions, + task=task, + exposure=exposure, + sample_weight=sample_weight, + metrics=metrics, + ) + model_names = tuple(point.results) + if reference is not None and ( + not isinstance(reference, str) or not reference.strip() + ): + raise InputValidationError("reference must be a non-empty model name.") + reference_name = model_names[0] if reference is None else reference.strip() + if reference_name not in point.results: + raise InputValidationError(f"Unknown reference model: {reference!r}.") + candidates = tuple(name for name in model_names if name != reference_name) + if not candidates: + raise InputValidationError("At least two models are required for comparison.") + validated_predictions: dict[str, NumericArray] = {} + baseline_inputs = None + for name, prediction in predictions.items(): + current = validate_inputs( + y_true, + prediction, + sample_weight=sample_weight, + exposure=exposure, + y_true_domain="nonnegative", + y_pred_domain="nonnegative", + ) + baseline_inputs = current + validated_predictions[name.strip()] = current.y_pred + assert baseline_inputs is not None + metric_names = tuple(point.results[reference_name].metrics) + samples: dict[tuple[str, str], list[float]] = { + (model, metric): [] for model in candidates for metric in metric_names + } + generator = np.random.default_rng(random_state) + attempts = 0 + max_attempts = max(count * 20, count + 100) + while len(next(iter(samples.values()))) < count and attempts < max_attempts: + attempts += 1 + indices = generator.integers( + 0, len(baseline_inputs.y_true), len(baseline_inputs.y_true) + ) + try: + result = compare( + baseline_inputs.y_true[indices], + { + name: prediction[indices] + for name, prediction in validated_predictions.items() + }, + task=task, + exposure=_slice_optional(baseline_inputs.exposure, indices), + sample_weight=_slice_optional(baseline_inputs.sample_weight, indices), + metrics=metrics, + ) + except ActEvalError: + continue + reference_result = result.results[reference_name] + if not all( + np.isfinite(metric_value) + for evaluation in result.results.values() + for metric_value in evaluation.metrics.values() + ): + continue + for model in candidates: + for metric in metric_names: + metadata = reference_result.metadata["metric_specs"][metric] + model_value = result.results[model].metrics[metric] + reference_value = reference_result.metrics[metric] + delta = _objective_value(model_value, metadata) - _objective_value( + reference_value, metadata + ) + samples[(model, metric)].append(delta) + completed = len(next(iter(samples.values()))) + if completed < count: + raise InputValidationError( + f"Only {completed} valid paired resamples could be generated after " + f"{attempts} attempts." + ) + comparisons: list[PairedMetricComparison] = [] + reference_result = point.results[reference_name] + for model in candidates: + model_result = point.results[model] + for metric in metric_names: + metadata = reference_result.metadata["metric_specs"][metric] + model_value = model_result.metrics[metric] + reference_value = reference_result.metrics[metric] + objective_delta = _objective_value( + model_value, metadata + ) - _objective_value(reference_value, metadata) + interval = _interval( + objective_delta, + np.asarray(samples[(model, metric)], dtype=np.float64), + confidence_level=confidence, + ) + comparisons.append( + PairedMetricComparison( + model=model, + reference=reference_name, + metric=metric, + model_estimate=model_value, + reference_estimate=reference_value, + raw_delta=model_value - reference_value, + objective_delta=objective_delta, + lower=interval.lower, + upper=interval.upper, + standard_error=interval.standard_error, + confidence_level=confidence, + n_resamples=count, + ) + ) + return PairedComparisonResult( + point.task, + reference_name, + tuple(comparisons), + { + "method": "paired_iid_percentile_bootstrap", + "n_resamples": count, + "confidence_level": confidence, + "random_state": random_state, + "attempts": attempts, + "failed_resamples": attempts - count, + "negative_objective_delta_favors_candidate": True, + "multiple_testing_adjustment": None, + }, + ) + + +def bootstrap_calibration_by_quantile( + y_true: ArrayLike, + y_pred: ArrayLike, + *, + n_bins: int = 10, + exposure: ArrayLike | None = None, + sample_weight: ArrayLike | None = None, + n_resamples: int = 1_000, + confidence_level: float = 0.95, + random_state: int | None = 0, +) -> CalibrationIntervalTable: + """Add stratified percentile intervals to fixed prediction-risk bins. + + Bins are defined once from the original predictions. Rows are then + resampled within each bin, so intervals describe conditional calibration + without allowing bootstrap cut points to change their interpretation. + """ + + count, confidence = _validate_bootstrap(n_resamples, confidence_level) + validated = validate_inputs( + y_true, + y_pred, + sample_weight=sample_weight, + exposure=exposure, + y_true_domain="nonnegative", + y_pred_domain="nonnegative", + ) + weights = combine_weights(validated) + effective = ( + np.ones(len(validated.y_true), dtype=np.float64) if weights is None else weights + ) + point_table = calibration_by_quantile( + validated.y_true, + validated.y_pred, + n_bins=n_bins, + exposure=validated.exposure, + sample_weight=validated.sample_weight, + ) + assignments = risk_bin_indices( + validated.y_pred, n_bins=n_bins, weights=effective + ).astype(np.intp) + generator = np.random.default_rng(random_state) + result_bins: list[CalibrationIntervalBin] = [] + total_attempts = 0 + for point_bin in point_table.bins: + members = np.flatnonzero((assignments == point_bin.bin) & (effective > 0)) + if not len(members): + continue + predicted_values: list[float] = [] + observed_values: list[float] = [] + ratio_values: list[float] = [] + attempts = 0 + max_attempts = max(count * 20, count + 100) + while len(ratio_values) < count and attempts < max_attempts: + attempts += 1 + selected = generator.choice(members, size=len(members), replace=True) + selected_weights = effective[selected] + weight_total = np.sum(selected_weights) + predicted_total = np.sum(selected_weights * validated.y_pred[selected]) + observed_total = np.sum(selected_weights * validated.y_true[selected]) + if weight_total <= 0 or predicted_total <= 0: + continue + predicted_values.append(float(predicted_total / weight_total)) + observed_values.append(float(observed_total / weight_total)) + ratio_values.append(float(observed_total / predicted_total)) + total_attempts += attempts + if len(ratio_values) < count: + raise InputValidationError( + f"Only {len(ratio_values)} valid calibration resamples could be " + f"generated for risk bin {point_bin.bin} after {attempts} attempts." + ) + predicted_samples = np.asarray(predicted_values, dtype=np.float64) + observed_samples = np.asarray(observed_values, dtype=np.float64) + ratio_samples = np.asarray(ratio_values, dtype=np.float64) + result_bins.append( + CalibrationIntervalBin( + bin=point_bin.bin, + count=point_bin.count, + mean_prediction=_interval( + point_bin.mean_prediction, + predicted_samples, + confidence_level=confidence, + ), + mean_observed=_interval( + point_bin.mean_observed, + observed_samples, + confidence_level=confidence, + ), + ae_ratio=_interval( + point_bin.ae_ratio, + ratio_samples, + confidence_level=confidence, + ), + ) + ) + return CalibrationIntervalTable( + tuple(result_bins), + n_bins, + confidence, + count, + { + "method": "fixed_bin_stratified_percentile_bootstrap", + "random_state": random_state, + "attempts": total_attempts, + "failed_resamples": total_attempts - count * len(result_bins), + "zero_effective_weight_rows_excluded": True, + }, + ) + + +def save_interval_csv( + result: BootstrapEvaluationResult + | PairedComparisonResult + | CalibrationIntervalTable, + path: str | Path, +) -> Path: + """Write an inference result table to CSV and return the resolved path.""" + + destination = Path(path).expanduser().resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + result.to_dataframe().to_csv(destination) + return destination diff --git a/src/acteval/monitoring.py b/src/acteval/monitoring.py new file mode 100644 index 0000000..bc6c7ed --- /dev/null +++ b/src/acteval/monitoring.py @@ -0,0 +1,480 @@ +"""Portfolio-segment, temporal, and prediction-drift diagnostics.""" + +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass +from typing import Any, cast + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +from acteval.api import MetricSelection, compare, evaluate +from acteval.exceptions import InputValidationError +from acteval.reports import ComparisonResult, EvaluationResult +from acteval.types import NumericArray, Task +from acteval.utils import weighted_quantile +from acteval.validation import combine_weights, validate_inputs + +ObjectArray = NDArray[np.object_] +BoolArray = NDArray[np.bool_] + + +@dataclass(frozen=True, slots=True) +class SegmentEvaluationResult: + """Metric results split by a caller-supplied portfolio segment.""" + + task: Task + segments: dict[str, EvaluationResult] + metadata: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-friendly representation.""" + + return { + "task": self.task, + "segments": { + segment: result.to_dict() for segment, result in self.segments.items() + }, + "metadata": dict(self.metadata), + } + + def to_dataframe(self) -> Any: + """Return metrics indexed by segment and metric.""" + + import pandas as pd + + rows = [ + {"segment": segment, "metric": metric, "value": value} + for segment, result in self.segments.items() + for metric, value in result.metrics.items() + ] + return pd.DataFrame.from_records(rows).set_index(["segment", "metric"]) + + def summary(self) -> str: + """Return a concise printable segment table.""" + + return str(self.to_dataframe().to_string()) + + +@dataclass(frozen=True, slots=True) +class SegmentComparisonResult: + """Multi-model comparisons repeated consistently within each segment.""" + + task: Task + segments: dict[str, ComparisonResult] + metadata: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-friendly representation.""" + + return { + "task": self.task, + "segments": { + segment: result.to_dict() for segment, result in self.segments.items() + }, + "metadata": dict(self.metadata), + } + + def to_dataframe(self) -> Any: + """Return segment and metric rows with one column per model.""" + + import pandas as pd + + frames = { + segment: comparison.to_dataframe() + for segment, comparison in self.segments.items() + } + result = pd.concat(frames, names=["segment", "metric"]) + return result + + def summary(self) -> str: + """Return a concise printable segmented comparison table.""" + + return str(self.to_dataframe().to_string()) + + +@dataclass(frozen=True, slots=True) +class TemporalEvaluationResult: + """Chronologically ordered metric results and changes from baseline.""" + + task: Task + periods: dict[str, EvaluationResult] + baseline_period: str + metadata: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-friendly representation.""" + + return { + "task": self.task, + "baseline_period": self.baseline_period, + "periods": { + period: result.to_dict() for period, result in self.periods.items() + }, + "metadata": dict(self.metadata), + } + + def to_dataframe(self) -> Any: + """Return period estimates and signed changes from the first period.""" + + import pandas as pd + + baseline = self.periods[self.baseline_period] + rows = [] + for period, result in self.periods.items(): + for metric, value in result.metrics.items(): + rows.append( + { + "period": period, + "metric": metric, + "value": value, + "baseline_value": baseline.metrics[metric], + "change_from_baseline": value - baseline.metrics[metric], + } + ) + return pd.DataFrame.from_records(rows).set_index(["period", "metric"]) + + def summary(self) -> str: + """Return a concise printable temporal table.""" + + return str(self.to_dataframe().to_string()) + + +@dataclass(frozen=True, slots=True) +class DriftBin: + """Reference and current portfolio shares in one fixed score bin.""" + + bin: int + lower: float + upper: float + reference_proportion: float + current_proportion: float + psi_contribution: float + + +@dataclass(frozen=True, slots=True) +class PredictionDriftResult: + """Distribution shift diagnostics for predictions across two populations.""" + + population_stability_index: float + reference_mean: float + current_mean: float + mean_shift: float + relative_mean_shift: float | None + bins: tuple[DriftBin, ...] + metadata: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-friendly representation.""" + + return { + "population_stability_index": self.population_stability_index, + "reference_mean": self.reference_mean, + "current_mean": self.current_mean, + "mean_shift": self.mean_shift, + "relative_mean_shift": self.relative_mean_shift, + "bins": [asdict(bin_result) for bin_result in self.bins], + "metadata": dict(self.metadata), + } + + def to_dataframe(self) -> Any: + """Return one row per fixed prediction bin.""" + + import pandas as pd + + return pd.DataFrame.from_records([asdict(row) for row in self.bins]).set_index( + "bin" + ) + + def summary(self) -> str: + """Return a concise printable drift report.""" + + header = ( + f"PSI={self.population_stability_index:.6g}, " + f"mean_shift={self.mean_shift:.6g}" + ) + return f"{header}\n{self.to_dataframe().to_string()}" + + +def _labels(values: ArrayLike, *, name: str, expected_length: int) -> ObjectArray: + import pandas as pd + + array = np.asarray(values, dtype=object) + if array.ndim != 1: + raise InputValidationError(f"{name} must be one-dimensional.") + if len(array) != expected_length: + raise InputValidationError( + f"{name} has length {len(array)}; expected {expected_length}." + ) + if any( + pd.isna(value) or (isinstance(value, str) and not value.strip()) + for value in array + ): + raise InputValidationError(f"{name} must not contain missing or empty labels.") + return cast(ObjectArray, array) + + +def _normalized_groups(values: ObjectArray) -> tuple[tuple[str, BoolArray], ...]: + labels: list[str] = [] + masks: list[BoolArray] = [] + seen: dict[str, object] = {} + for value in values: + label = str(value).strip() + if label in seen and seen[label] != value: + raise InputValidationError( + f"Distinct group values collapse to the same label {label!r}." + ) + if label not in seen: + seen[label] = value + labels.append(label) + masks.append(np.asarray(values == value, dtype=np.bool_)) + return tuple((label, mask) for label, mask in zip(labels, masks, strict=True)) + + +def _slice(values: ArrayLike | None, mask: BoolArray) -> ArrayLike | None: + if values is None: + return None + return np.asarray(values)[mask] + + +def _validate_min_observations(value: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise InputValidationError("min_observations must be a positive integer.") + return value + + +def evaluate_by_segment( + y_true: ArrayLike, + y_pred: ArrayLike, + segments: ArrayLike, + *, + task: str, + exposure: ArrayLike | None = None, + sample_weight: ArrayLike | None = None, + metrics: Sequence[MetricSelection] | None = None, + min_observations: int = 2, +) -> SegmentEvaluationResult: + """Evaluate one model separately within each portfolio segment.""" + + validated = validate_inputs( + y_true, + y_pred, + sample_weight=sample_weight, + exposure=exposure, + y_true_domain="nonnegative", + y_pred_domain="nonnegative", + ) + minimum = _validate_min_observations(min_observations) + labels = _labels(segments, name="segments", expected_length=len(validated.y_true)) + results: dict[str, EvaluationResult] = {} + skipped: list[str] = [] + for label, mask in _normalized_groups(labels): + if np.count_nonzero(mask) < minimum: + skipped.append(label) + continue + results[label] = evaluate( + validated.y_true[mask], + validated.y_pred[mask], + task=task, + exposure=_slice(validated.exposure, mask), + sample_weight=_slice(validated.sample_weight, mask), + metrics=metrics, + ) + if not results: + raise InputValidationError("No segment meets min_observations.") + first = next(iter(results.values())) + return SegmentEvaluationResult( + first.task, + results, + { + "n_segments": len(results), + "min_observations": minimum, + "skipped_segments": tuple(skipped), + }, + ) + + +def compare_by_segment( + y_true: ArrayLike, + predictions: Mapping[str, ArrayLike], + segments: ArrayLike, + *, + task: str, + exposure: ArrayLike | None = None, + sample_weight: ArrayLike | None = None, + metrics: Sequence[MetricSelection] | None = None, + min_observations: int = 2, +) -> SegmentComparisonResult: + """Compare models within each portfolio segment using identical rows.""" + + if not predictions: + raise InputValidationError("predictions must contain at least one model.") + minimum = _validate_min_observations(min_observations) + first_prediction = next(iter(predictions.values())) + validated = validate_inputs( + y_true, + first_prediction, + sample_weight=sample_weight, + exposure=exposure, + y_true_domain="nonnegative", + y_pred_domain="nonnegative", + ) + labels = _labels(segments, name="segments", expected_length=len(validated.y_true)) + results: dict[str, ComparisonResult] = {} + skipped: list[str] = [] + for label, mask in _normalized_groups(labels): + if np.count_nonzero(mask) < minimum: + skipped.append(label) + continue + results[label] = compare( + validated.y_true[mask], + {name: np.asarray(values)[mask] for name, values in predictions.items()}, + task=task, + exposure=_slice(validated.exposure, mask), + sample_weight=_slice(validated.sample_weight, mask), + metrics=metrics, + ) + if not results: + raise InputValidationError("No segment meets min_observations.") + first = next(iter(results.values())) + return SegmentComparisonResult( + first.task, + results, + { + "n_segments": len(results), + "min_observations": minimum, + "skipped_segments": tuple(skipped), + "no_universal_best_model": True, + }, + ) + + +def evaluate_over_time( + y_true: ArrayLike, + y_pred: ArrayLike, + periods: ArrayLike, + *, + task: str, + exposure: ArrayLike | None = None, + sample_weight: ArrayLike | None = None, + metrics: Sequence[MetricSelection] | None = None, + min_observations: int = 2, +) -> TemporalEvaluationResult: + """Evaluate chronologically sorted periods and changes from the first period. + + Period labels should sort chronologically as strings; ISO dates and ordered + year/month or quarter labels satisfy this contract. + """ + + segmented = evaluate_by_segment( + y_true, + y_pred, + periods, + task=task, + exposure=exposure, + sample_weight=sample_weight, + metrics=metrics, + min_observations=min_observations, + ) + ordered = dict(sorted(segmented.segments.items())) + baseline = next(iter(ordered)) + return TemporalEvaluationResult( + segmented.task, + ordered, + baseline, + { + **segmented.metadata, + "period_order": tuple(ordered), + "change_is_signed_current_minus_baseline": True, + }, + ) + + +def _weighted_mean(values: NumericArray, weights: NumericArray | None) -> float: + return float( + np.mean(values) if weights is None else np.average(values, weights=weights) + ) + + +def prediction_drift( + reference_predictions: ArrayLike, + current_predictions: ArrayLike, + *, + n_bins: int = 10, + reference_weight: ArrayLike | None = None, + current_weight: ArrayLike | None = None, + epsilon: float = 1e-6, +) -> PredictionDriftResult: + """Measure score-distribution drift with fixed reference quantile bins. + + PSI is descriptive and has no universal alert threshold. ActEval therefore + returns the value and bin contributions without labelling drift as good, + bad, significant, or operationally material. + """ + + reference = validate_inputs( + reference_predictions, + reference_predictions, + sample_weight=reference_weight, + ) + current = validate_inputs( + current_predictions, + current_predictions, + sample_weight=current_weight, + ) + if isinstance(n_bins, bool) or not isinstance(n_bins, int) or n_bins < 2: + raise InputValidationError("n_bins must be an integer of at least 2.") + if not np.isfinite(epsilon) or epsilon <= 0: + raise InputValidationError("epsilon must be strictly positive and finite.") + reference_weights = combine_weights(reference) + current_weights = combine_weights(current) + quantiles = np.linspace(0, 1, n_bins + 1)[1:-1] + cut_points = np.unique( + [ + weighted_quantile(reference.y_true, float(q), reference_weights) + for q in quantiles + ] + ) + edges = np.concatenate(([-np.inf], cut_points, [np.inf])) + reference_counts, _ = np.histogram( + reference.y_true, bins=edges, weights=reference_weights + ) + current_counts, _ = np.histogram( + current.y_true, bins=edges, weights=current_weights + ) + reference_proportions = reference_counts / np.sum(reference_counts) + current_proportions = current_counts / np.sum(current_counts) + reference_safe = np.clip(reference_proportions, epsilon, None) + current_safe = np.clip(current_proportions, epsilon, None) + contributions = (current_safe - reference_safe) * np.log( + current_safe / reference_safe + ) + bins = tuple( + DriftBin( + bin=index + 1, + lower=float(edges[index]), + upper=float(edges[index + 1]), + reference_proportion=float(reference_proportions[index]), + current_proportion=float(current_proportions[index]), + psi_contribution=float(contributions[index]), + ) + for index in range(len(edges) - 1) + ) + reference_mean = _weighted_mean(reference.y_true, reference_weights) + current_mean = _weighted_mean(current.y_true, current_weights) + shift = current_mean - reference_mean + relative = shift / reference_mean if reference_mean != 0 else None + return PredictionDriftResult( + population_stability_index=float(np.sum(contributions)), + reference_mean=reference_mean, + current_mean=current_mean, + mean_shift=shift, + relative_mean_shift=relative, + bins=bins, + metadata={ + "requested_bins": n_bins, + "effective_bins": len(bins), + "epsilon": epsilon, + "psi_has_no_universal_threshold": True, + "binning": "reference_prediction_quantiles", + }, + ) diff --git a/src/acteval/reporting.py b/src/acteval/reporting.py new file mode 100644 index 0000000..0f8e366 --- /dev/null +++ b/src/acteval/reporting.py @@ -0,0 +1,125 @@ +"""Dependency-light export and standalone HTML reporting utilities.""" + +import json +from html import escape +from pathlib import Path +from typing import Any, Protocol + + +class TabularResult(Protocol): + """Structural contract used by result export helpers.""" + + def to_dataframe(self) -> Any: + """Return the primary result table.""" + ... + + def to_dict(self) -> Any: + """Return JSON-friendly result content.""" + ... + + +_STYLE = """ +body { font-family: ui-sans-serif, system-ui, sans-serif; margin: 2rem auto; + max-width: 1100px; padding: 0 1rem; color: #172033; } +h1 { color: #075985; } h2 { margin-top: 2rem; } +table { border-collapse: collapse; width: 100%; font-size: 0.92rem; } +th, td { border: 1px solid #d8dee9; padding: 0.5rem 0.65rem; text-align: right; } +th { background: #f0f7fa; } tbody tr:nth-child(even) { background: #f8fafc; } +pre { background: #f6f8fa; border: 1px solid #d8dee9; border-radius: 6px; + padding: 1rem; overflow-x: auto; } +.note { border-left: 4px solid #0284c7; padding: 0.75rem 1rem; background: #f0f9ff; } +""".strip() + + +def render_html_report( + result: TabularResult, + *, + title: str = "ActEval report", + include_metadata: bool = True, +) -> str: + """Render any tabular ActEval result as a standalone UTF-8 HTML document. + + The report has no JavaScript or remote assets, so it can be archived and + reviewed offline. Content generated from labels and metadata is escaped. + """ + + frame = result.to_dataframe() + table = frame.to_html(border=0, escape=True, classes="acteval-table") + metadata_html = "" + payload = result.to_dict() + if include_metadata: + metadata = payload.get("metadata", {}) if isinstance(payload, dict) else {} + metadata_html = ( + "

Reproducibility metadata

"
+            + escape(json.dumps(metadata, indent=2, sort_keys=True, default=str))
+            + "
" + ) + safe_title = escape(title) + return ( + '' + '' + f"{safe_title}" + f"

{safe_title}

" + '

Metrics represent distinct actuarial objectives; ' + "this report does not define a universal best model.

" + f"

Results

{table}{metadata_html}" + ) + + +def save_html_report( + result: TabularResult, + path: str | Path, + *, + title: str = "ActEval report", + include_metadata: bool = True, +) -> Path: + """Write a standalone HTML report and return its resolved path.""" + + destination = Path(path).expanduser().resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + render_html_report(result, title=title, include_metadata=include_metadata), + encoding="utf-8", + ) + return destination + + +def export_table( + result: TabularResult, + path: str | Path, + *, + format: str | None = None, +) -> Path: + """Export a result table to CSV, JSON, or standalone HTML. + + The format defaults to the destination extension. JSON uses the full + structured result while CSV exports the primary table. + """ + + destination = Path(path).expanduser().resolve() + resolved_format = (format or destination.suffix.lstrip(".")).strip().lower() + if resolved_format not in {"csv", "json", "html"}: + raise ValueError("format must be one of: csv, json, html.") + destination.parent.mkdir(parents=True, exist_ok=True) + if resolved_format == "csv": + result.to_dataframe().to_csv(destination) + elif resolved_format == "json": + destination.write_text( + json.dumps(result.to_dict(), indent=2, default=str), encoding="utf-8" + ) + else: + destination.write_text(render_html_report(result), encoding="utf-8") + return destination + + +def save_plot(axis: Any, path: str | Path, *, dpi: int = 150) -> Path: + """Save a Matplotlib axis' figure with a tight bounding box.""" + + if isinstance(dpi, bool) or not isinstance(dpi, int) or dpi < 1: + raise ValueError("dpi must be a positive integer.") + if not hasattr(axis, "figure") or not hasattr(axis.figure, "savefig"): + raise TypeError("axis must be a Matplotlib axis with a figure.") + destination = Path(path).expanduser().resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + axis.figure.savefig(destination, dpi=dpi, bbox_inches="tight") + return destination diff --git a/src/acteval/reports/comparison.py b/src/acteval/reports/comparison.py index 4231b42..8ee76e7 100644 --- a/src/acteval/reports/comparison.py +++ b/src/acteval/reports/comparison.py @@ -1,6 +1,7 @@ """Multi-model evaluation result.""" from dataclasses import dataclass +from pathlib import Path from typing import Any from acteval.exceptions import InputValidationError @@ -72,3 +73,19 @@ def rank(self, metric: str) -> Any: ranking = ranking.reset_index(drop=True) ranking["rank"] = ranking.index + 1 return ranking + + def to_html(self, *, title: str = "ActEval model comparison") -> str: + """Render this comparison as a standalone HTML report.""" + + from acteval.reporting import render_html_report + + return render_html_report(self, title=title) + + def save_html( + self, path: str | Path, *, title: str = "ActEval model comparison" + ) -> Path: + """Write a standalone HTML report and return its resolved path.""" + + from acteval.reporting import save_html_report + + return save_html_report(self, path, title=title) diff --git a/src/acteval/reports/result.py b/src/acteval/reports/result.py index 43f822b..a00d416 100644 --- a/src/acteval/reports/result.py +++ b/src/acteval/reports/result.py @@ -2,6 +2,7 @@ from copy import deepcopy from dataclasses import dataclass +from pathlib import Path from typing import Any from acteval.types import Task @@ -46,3 +47,17 @@ def to_dataframe(self) -> Any: def summary(self) -> str: """Return a concise printable metric table.""" return str(self.to_dataframe().to_string()) + + def to_html(self, *, title: str = "ActEval evaluation") -> str: + """Render this result as a standalone HTML report.""" + + from acteval.reporting import render_html_report + + return render_html_report(self, title=title) + + def save_html(self, path: str | Path, *, title: str = "ActEval evaluation") -> Path: + """Write a standalone HTML report and return its resolved path.""" + + from acteval.reporting import save_html_report + + return save_html_report(self, path, title=title) diff --git a/tests/test_api.py b/tests/test_api.py index 9f9cd87..d75c550 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -116,3 +116,25 @@ def test_evaluate_rejects_unsupported_and_duplicate_metrics() -> None: def test_compare_rejects_empty_predictions() -> None: with pytest.raises(InputValidationError, match="at least one model"): ae.compare([1, 2], {}, task="claim_frequency") + + +def test_public_api_exports_are_resolvable_and_version_is_stable() -> None: + assert ae.__version__ == "1.0.0" + assert ae.__all__ + assert len(ae.__all__) == len(set(ae.__all__)) + assert all(hasattr(ae, name) for name in ae.__all__) + + +def test_metric_signature_validation_is_cached( + frequency_data: tuple[np.ndarray, np.ndarray, np.ndarray], +) -> None: + from acteval.api import _metric_signature + + observed, predicted, _ = frequency_data + _metric_signature.cache_clear() + ae.evaluate(observed, predicted, task="claim_frequency", metrics=["rmse"]) + first = _metric_signature.cache_info() + ae.evaluate(observed, predicted, task="claim_frequency", metrics=["rmse"]) + second = _metric_signature.cache_info() + assert first.misses == 1 + assert second.hits == first.hits + 1 diff --git a/tests/test_examples.py b/tests/test_examples.py index fbbf9f0..a52b529 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -24,6 +24,18 @@ def test_decision_example_executes(capsys: object) -> None: assert "Reinsurance selection:" in output +def test_inference_and_monitoring_example_executes( + capsys: object, tmp_path: Path, monkeypatch: object +) -> None: + monkeypatch.chdir(tmp_path) # type: ignore[attr-defined] + example = Path(__file__).parents[1] / "examples" / "inference_and_monitoring.py" + runpy.run_path(str(example), run_name="__main__") + output = capsys.readouterr().out # type: ignore[attr-defined] + assert "objective_delta" not in output + assert "PSI=" in output + assert (tmp_path / "acteval-example-report.html").exists() + + def test_example_notebook_is_valid_notebook_json() -> None: notebook = json.loads( Path("examples/synthetic_frequency.ipynb").read_text(encoding="utf-8") diff --git a/tests/test_inference.py b/tests/test_inference.py new file mode 100644 index 0000000..a09f96e --- /dev/null +++ b/tests/test_inference.py @@ -0,0 +1,226 @@ +from pathlib import Path + +import numpy as np +import pytest + +import acteval as ae +from acteval.exceptions import InputValidationError + + +@pytest.fixture +def inference_data() -> tuple[np.ndarray, np.ndarray, np.ndarray]: + rng = np.random.default_rng(42) + predicted = np.linspace(0.2, 5.0, 80) + observed = rng.poisson(predicted).astype(float) + exposure = np.linspace(0.5, 1.5, len(observed)) + return observed, predicted, exposure + + +def test_bootstrap_evaluate_is_reproducible_and_contains_point_estimate( + inference_data: tuple[np.ndarray, np.ndarray, np.ndarray], +) -> None: + observed, predicted, exposure = inference_data + first = ae.bootstrap_evaluate( + observed, + predicted, + task="claim_frequency", + exposure=exposure, + metrics=["rmse", "ae_ratio"], + n_resamples=100, + random_state=17, + ) + second = ae.bootstrap_evaluate( + observed, + predicted, + task="claim_frequency", + exposure=exposure, + metrics=["rmse", "ae_ratio"], + n_resamples=100, + random_state=17, + ) + assert first.to_dict() == second.to_dict() + assert first.intervals["rmse"].estimate == pytest.approx( + ae.rmse(observed, predicted, exposure=exposure) + ) + assert first.intervals["rmse"].lower <= first.intervals["rmse"].upper + assert first.to_dataframe().shape == (2, 7) + assert "rmse" in first.summary() + + +def test_paired_bootstrap_compare_uses_objective_aware_delta( + inference_data: tuple[np.ndarray, np.ndarray, np.ndarray], +) -> None: + observed, predicted, _ = inference_data + result = ae.paired_bootstrap_compare( + observed, + {"reference": predicted * 1.8, "candidate": predicted}, + task="claim_frequency", + reference="reference", + metrics=["rmse", "ae_ratio"], + n_resamples=100, + random_state=5, + ) + rmse = next(item for item in result.comparisons if item.metric == "rmse") + assert rmse.objective_delta < 0 + assert rmse.raw_delta < 0 + assert result.metadata["negative_objective_delta_favors_candidate"] is True + assert result.to_dataframe().shape[0] == 2 + + +def test_identical_models_have_zero_paired_interval( + inference_data: tuple[np.ndarray, np.ndarray, np.ndarray], +) -> None: + observed, predicted, _ = inference_data + result = ae.paired_bootstrap_compare( + observed, + {"A": predicted, "B": predicted.copy()}, + task="claim_frequency", + metrics=["rmse"], + n_resamples=100, + random_state=8, + ) + comparison = result.comparisons[0] + assert comparison.objective_delta == pytest.approx(0) + assert comparison.lower == pytest.approx(0) + assert comparison.upper == pytest.approx(0) + assert comparison.confidence_excludes_zero is False + + +def test_bootstrap_calibration_uses_fixed_bins_and_intervals( + inference_data: tuple[np.ndarray, np.ndarray, np.ndarray], +) -> None: + observed, predicted, exposure = inference_data + result = ae.bootstrap_calibration_by_quantile( + observed, + predicted, + n_bins=5, + exposure=exposure, + n_resamples=100, + random_state=3, + ) + assert result.requested_bins == 5 + assert result.effective_bins == 5 + assert result.to_dataframe().shape == (5, 10) + for row in result.bins: + assert row.mean_observed.lower <= row.mean_observed.upper + assert row.ae_ratio.n_resamples == 100 + assert result.metadata["method"] == "fixed_bin_stratified_percentile_bootstrap" + + +def test_bootstrap_calibration_retries_zero_predicted_resamples() -> None: + observed = [0, 0, 0, 1, 2, 3, 4, 5] + predicted = [0, 0, 0, 0.1, 1, 2, 3, 4] + result = ae.bootstrap_calibration_by_quantile( + observed, + predicted, + n_bins=2, + n_resamples=100, + random_state=4, + ) + assert result.metadata["failed_resamples"] >= 0 + + +def test_bootstrap_supports_discrimination_lift_and_tail_metrics( + inference_data: tuple[np.ndarray, np.ndarray, np.ndarray], +) -> None: + observed, predicted, exposure = inference_data + result = ae.bootstrap_evaluate( + observed, + predicted, + task="claim_frequency", + exposure=exposure, + metrics=["normalized_gini", "lift", "tail_mae_80", "tail_ae_80"], + n_resamples=100, + random_state=19, + ) + assert set(result.intervals) == { + "normalized_gini", + "lift", + "tail_mae_80", + "tail_ae_80", + } + assert all(interval.n_resamples == 100 for interval in result.intervals.values()) + + +def test_paired_bootstrap_reverses_higher_is_better_objective() -> None: + observed = np.arange(1, 61, dtype=float) + candidate = observed.copy() + reference = observed[::-1].copy() + result = ae.paired_bootstrap_compare( + observed, + {"reference": reference, "candidate": candidate}, + task="claim_frequency", + reference="reference", + metrics=["normalized_gini"], + n_resamples=100, + random_state=12, + ) + comparison = result.comparisons[0] + assert comparison.raw_delta > 0 + assert comparison.objective_delta < 0 + + +def test_save_interval_csv( + tmp_path: Path, + inference_data: tuple[np.ndarray, np.ndarray, np.ndarray], +) -> None: + observed, predicted, _ = inference_data + result = ae.bootstrap_evaluate( + observed, + predicted, + task="claim_frequency", + metrics=["rmse"], + n_resamples=100, + ) + destination = ae.save_interval_csv(result, tmp_path / "nested" / "intervals.csv") + assert destination.exists() + assert "standard_error" in destination.read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"n_resamples": 99}, "at least 100"), + ({"confidence_level": 1.0}, "strictly between"), + ], +) +def test_bootstrap_rejects_invalid_configuration( + kwargs: dict[str, float | int], message: str +) -> None: + with pytest.raises(InputValidationError, match=message): + ae.bootstrap_evaluate( + [1, 2, 3], + [1, 2, 3], + task="claim_frequency", + metrics=["rmse"], + **kwargs, + ) + + +def test_paired_comparison_rejects_unknown_or_only_reference() -> None: + with pytest.raises(InputValidationError, match="Unknown reference"): + ae.paired_bootstrap_compare( + [1, 2, 3], + {"A": [1, 2, 3], "B": [1, 2, 3]}, + task="claim_frequency", + reference="missing", + metrics=["rmse"], + n_resamples=100, + ) + with pytest.raises(InputValidationError, match="non-empty"): + ae.paired_bootstrap_compare( + [1, 2, 3], + {"A": [1, 2, 3], "B": [1, 2, 3]}, + task="claim_frequency", + reference="", + metrics=["rmse"], + n_resamples=100, + ) + with pytest.raises(InputValidationError, match="At least two"): + ae.paired_bootstrap_compare( + [1, 2, 3], + {"A": [1, 2, 3]}, + task="claim_frequency", + metrics=["rmse"], + n_resamples=100, + ) diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py new file mode 100644 index 0000000..a7162ac --- /dev/null +++ b/tests/test_monitoring.py @@ -0,0 +1,133 @@ +import numpy as np +import pytest + +import acteval as ae +from acteval.exceptions import InputValidationError + + +@pytest.fixture +def monitoring_data() -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + observed = np.asarray([0, 1, 2, 1, 3, 4, 2, 5, 6, 4, 7, 8], dtype=float) + predicted = observed * 0.9 + 0.2 + segments = np.asarray(["Retail"] * 6 + ["Commercial"] * 6) + periods = np.asarray(["2025-Q2"] * 4 + ["2025-Q1"] * 4 + ["2025-Q3"] * 4) + return observed, predicted, segments, periods + + +def test_evaluate_and_compare_by_segment( + monitoring_data: tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray], +) -> None: + observed, predicted, segments, _ = monitoring_data + result = ae.evaluate_by_segment( + observed, + predicted, + segments, + task="claim_frequency", + metrics=["rmse", "ae_ratio"], + ) + assert set(result.segments) == {"Retail", "Commercial"} + assert result.to_dataframe().shape == (4, 1) + comparison = ae.compare_by_segment( + observed, + {"A": predicted, "B": predicted * 1.1}, + segments, + task="claim_frequency", + metrics=["rmse"], + ) + assert comparison.to_dataframe().shape == (2, 2) + assert comparison.metadata["no_universal_best_model"] is True + + +def test_segment_evaluation_skips_small_groups( + monitoring_data: tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray], +) -> None: + observed, predicted, _, _ = monitoring_data + segments = np.asarray(["small"] + ["large"] * 11) + result = ae.evaluate_by_segment( + observed, + predicted, + segments, + task="claim_frequency", + metrics=["rmse"], + min_observations=2, + ) + assert set(result.segments) == {"large"} + assert result.metadata["skipped_segments"] == ("small",) + + +def test_temporal_evaluation_sorts_periods_and_reports_change( + monitoring_data: tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray], +) -> None: + observed, predicted, _, periods = monitoring_data + result = ae.evaluate_over_time( + observed, + predicted, + periods, + task="claim_frequency", + metrics=["rmse"], + ) + assert tuple(result.periods) == ("2025-Q1", "2025-Q2", "2025-Q3") + assert result.baseline_period == "2025-Q1" + baseline_row = result.to_dataframe().loc[("2025-Q1", "rmse")] + assert baseline_row["change_from_baseline"] == pytest.approx(0) + + +def test_prediction_drift_known_behavior_and_weights() -> None: + reference = np.linspace(0, 1, 100) + identical = ae.prediction_drift(reference, reference.copy(), n_bins=5) + assert identical.population_stability_index == pytest.approx(0) + shifted = ae.prediction_drift( + reference, + reference + 0.5, + n_bins=5, + reference_weight=np.ones(100), + current_weight=np.linspace(1, 2, 100), + ) + assert shifted.population_stability_index > 0 + assert shifted.mean_shift > 0 + assert sum(row.psi_contribution for row in shifted.bins) == pytest.approx( + shifted.population_stability_index + ) + assert "PSI=" in shifted.summary() + + +def test_monitoring_rejects_invalid_inputs( + monitoring_data: tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray], +) -> None: + observed, predicted, segments, _ = monitoring_data + with pytest.raises(InputValidationError, match="expected 12"): + ae.evaluate_by_segment( + observed, + predicted, + segments[:-1], + task="claim_frequency", + metrics=["rmse"], + ) + with pytest.raises(InputValidationError, match="No segment"): + ae.evaluate_by_segment( + observed, + predicted, + segments, + task="claim_frequency", + metrics=["rmse"], + min_observations=100, + ) + with pytest.raises(InputValidationError, match="missing or empty"): + ae.evaluate_by_segment( + observed, + predicted, + [None, *list(segments[1:])], + task="claim_frequency", + metrics=["rmse"], + ) + with pytest.raises(InputValidationError, match="positive integer"): + ae.compare_by_segment( + observed, + {"A": predicted, "B": predicted}, + segments, + task="claim_frequency", + metrics=["rmse"], + min_observations=0, + ) + with pytest.raises(InputValidationError, match="epsilon"): + ae.prediction_drift([1, 2], [1, 2], epsilon=0) diff --git a/tests/test_reporting.py b/tests/test_reporting.py new file mode 100644 index 0000000..92c1ffc --- /dev/null +++ b/tests/test_reporting.py @@ -0,0 +1,55 @@ +import json +from pathlib import Path + +import matplotlib.pyplot as plt +import pytest + +import acteval as ae + + +def test_standalone_html_report_escapes_title_and_contains_metadata( + tmp_path: Path, +) -> None: + result = ae.evaluate( + [1, 2, 3], [1, 2, 2.5], task="claim_frequency", metrics=["rmse"] + ) + html = result.to_html(title="") + assert "" in html + assert "<ActEval & report>" in html + assert "Reproducibility metadata" in html + assert "universal best model" in html + destination = result.save_html(tmp_path / "reports" / "evaluation.html") + assert destination.exists() + assert destination.read_text(encoding="utf-8").startswith("") + + +def test_export_table_csv_json_and_html(tmp_path: Path) -> None: + comparison = ae.compare( + [1, 2, 3], + {"A": [1, 2, 3], "B": [1.1, 1.9, 2.8]}, + task="claim_frequency", + metrics=["rmse"], + ) + csv_path = ae.export_table(comparison, tmp_path / "comparison.csv") + json_path = ae.export_table(comparison, tmp_path / "comparison.json") + html_path = ae.export_table(comparison, tmp_path / "comparison.html") + assert "metric" in csv_path.read_text(encoding="utf-8") + assert json.loads(json_path.read_text(encoding="utf-8"))["task"] == ( + "claim_frequency" + ) + assert "" in html_path.read_text(encoding="utf-8") + assert "Model A" not in comparison.to_html() + with pytest.raises(ValueError, match="csv, json, html"): + ae.export_table(comparison, tmp_path / "bad.txt") + + +def test_save_plot(tmp_path: Path) -> None: + axis = ae.plot_residuals([1, 2, 3], [1.1, 1.9, 3.2]) + destination = ae.save_plot(axis, tmp_path / "plot.png", dpi=100) + assert destination.exists() + assert destination.stat().st_size > 0 + plt.close(axis.figure) + with pytest.raises(ValueError, match="positive integer"): + ae.save_plot(axis, tmp_path / "bad.png", dpi=0) + with pytest.raises(TypeError, match="Matplotlib axis"): + ae.save_plot(object(), tmp_path / "bad.png") From 1933a08fec82559592114de714444609b71585c9 Mon Sep 17 00:00:00 2001 From: aminemanai2003 Date: Wed, 12 Aug 2026 16:52:02 +0100 Subject: [PATCH 2/2] Finalize ActEval 1.0 release notes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9506e22..f2aba12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 1.0.0 - Unreleased +## 1.0.0 - 2026-08-12 ### Stable API and packaging