diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bb28c3b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: ci + +on: + push: + pull_request: + +jobs: + pipeline: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements*.txt + + - name: Install dependencies + run: pip install -r requirements-dev.txt + + - name: Data, train, evaluate, sanity check + run: make all + + - name: Data is reproducible from the seed + run: git diff --exit-code data/ + + - name: Unit tests + run: make test diff --git a/Makefile b/Makefile index 72b660e..c459168 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: data train evaluate sanity app all clean +.PHONY: data train evaluate sanity test app all clean PYTHON ?= python @@ -14,6 +14,9 @@ evaluate: sanity: $(PYTHON) -m src.models.sanity_check +test: + $(PYTHON) -m pytest -q tests + app: streamlit run app.py diff --git a/README.md b/README.md index f717304..c312746 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@

Live demo + CI ML Models

@@ -17,11 +18,14 @@ An ML-powered dynamic pricing and decision support system for ticket pricing in ## Quickstart ```bash -pip install -r requirements.txt +pip install -r requirements-dev.txt # Python 3.12, pinned versions make all # synthetic data, train, holdout evaluation, elasticity sanity check +make test # unit tests (optimizer bounds, cold start, save/load) make app # open the human-in-the-loop Streamlit page locally ``` +CI runs `make all` and `make test` on every push. The live demo sleeps after 12 hours without traffic (a Streamlit Community Cloud rule), so the first visit may need a click to wake it and about a minute to train. + --- ## The problem @@ -83,7 +87,7 @@ The **Dynamic Pricing Engine** ingests historical data from the **Club's Data Sy | Naive: last 7 days per series | 37.4% | 0.544 | 8.7 | 15.2 | 1.00 | | Naive: global mean | 79.7% | -0.428 | 18.6 | 27.0 | 2.13 | -The baseline that matters is the last-7-days naive: the forecast anyone could make at the cutoff without a model. The ensemble's WAPE is **29% lower** than it. The XGBoost stage does the heavy lifting: on its own, trained directly on sales, it beats the ensemble on WAPE and MAE, while the ensemble edges it on R² and RMSE (fewer large misses). Prophet alone is worse than the naive. Reproducible with `RANDOM_SEED=42` in `src/data/make_dataset.py`. +The baseline that matters is the last-7-days naive: the forecast anyone could make at the cutoff without a model. The ensemble's WAPE is **29% lower** than it. The XGBoost stage does the heavy lifting: on its own, trained directly on sales, it beats the ensemble on WAPE and MAE, while the ensemble edges it on R² and RMSE (fewer large misses). Prophet alone is worse than the naive. Reproducible with `RANDOM_SEED=42` in `src/data/make_dataset.py` and the pinned versions in `requirements.txt`. #### Cold start @@ -218,7 +222,9 @@ dynamic-pricing/ ├── README.md ├── app.py # Streamlit HiTL page ├── config.py # Paths and reference dates -├── requirements.txt +├── requirements.txt # Pinned runtime dependencies +├── requirements-dev.txt # + pytest +├── tests/ # Unit tests: optimizer bounds, cold start, save/load ├── assets/ # Diagrams and images ├── data/ │ └── 03_synthetic/ @@ -248,9 +254,10 @@ dynamic-pricing/ ### Reproducing the results ```bash -pip install -r requirements.txt +pip install -r requirements-dev.txt make clean # remove cached artifacts -make all # data + train + evaluate (accuracy + decision quality) + sanity check +make all # data + train + evaluate (accuracy, decisions, cold start) + sanity check +make test # unit tests make app # launch the Streamlit page # Or run the CLI examples directly: diff --git a/app.py b/app.py index 681a44e..6c6a72d 100644 --- a/app.py +++ b/app.py @@ -2,7 +2,7 @@ import json import os -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path import pandas as pd @@ -128,7 +128,7 @@ def load_data() -> pd.DataFrame: st.divider() if st.button("✓ Approve recommended price", type="primary"): proposal = { - 'timestamp': datetime.utcnow().isoformat(), + 'timestamp': datetime.now(UTC).isoformat(), 'match_id': int(match_id), 'seat_zone': seat_zone, 'days_until_match': int(days_until_match), diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..1441c92 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest==9.1.1 diff --git a/requirements.txt b/requirements.txt index 8bdf406..53196f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,8 @@ -pandas>=2.2.0 -numpy>=2.0.0 -joblib>=1.4.0 -scikit-learn>=1.5.0 -prophet>=1.1.5 -xgboost>=2.0.0 -streamlit>=1.36.0 -jupyterlab==4.2.1 -matplotlib -seaborn +# Pinned to the versions the README numbers were produced with (Python 3.12). +pandas==3.0.6 +numpy==2.5.3 +joblib==1.6.0 +scikit-learn==1.9.1 +prophet==1.5.0 +xgboost==3.4.1 +streamlit==1.65.0 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..02b91c8 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +import os +import sys + +import pandas as pd +import pytest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +import config +from src.models.train_demand_model import train_ensemble + + +@pytest.fixture(scope='session') +def data() -> pd.DataFrame: + return pd.read_csv(config.SYNTHETIC_DATA_PATH) + + +@pytest.fixture(scope='session') +def small_model(data): + """Ensemble trained on three matches: enough series to exercise every path, fast enough for CI.""" + return train_ensemble(data[data['match_id'].isin([1, 2, 3])]) diff --git a/tests/test_engine.py b/tests/test_engine.py new file mode 100644 index 0000000..9373ebd --- /dev/null +++ b/tests/test_engine.py @@ -0,0 +1,56 @@ +import numpy as np +import pytest + +import config +from src.data.make_dataset import price_effect +from src.decision_engine.constants import ZONE_BASE_PRICES +from src.decision_engine.optimize import OptimizationEngine +from src.models.evaluate import true_revenue_share +from src.models.train_demand_model import price_support + + +def test_true_optimum_is_one_and_a_quarter_times_base(): + ratios = np.linspace(0.5, 2.5, 2001) + assert ratios[np.argmax(ratios * price_effect(ratios))] == pytest.approx(1.25, abs=1e-3) + + +def test_price_support_is_inside_observed_prices(data): + for zone, (lo, hi) in price_support(data).items(): + prices = data.loc[data['seat_zone'] == zone, 'ticket_price'] + assert prices.min() <= lo < hi <= prices.max() + 1 + + +def test_recommendations_stay_inside_the_band(data, small_model): + rows = data[data['match_id'].isin([1, 2, 3])].drop(columns=[config.TARGET_COLUMN]).sample(30, random_state=0) + prices = OptimizationEngine(model=small_model).recommend_prices(rows) + for price, zone in zip(prices, rows['seat_zone']): + lo, hi = small_model.price_support[zone] + assert lo <= price <= hi + + +def test_batch_recommendation_matches_single_row_curve(data, small_model): + engine = OptimizationEngine(model=small_model) + row = data[data['match_id'] == 2].drop(columns=[config.TARGET_COLUMN]).iloc[[10]].reset_index(drop=True) + curve = engine.revenue_curve(row) + assert engine.recommend_prices(row)[0] == curve.loc[curve['projected_revenue'].idxmax(), 'price'] + + +def test_unseen_match_uses_the_pooled_model(data, small_model): + unseen = data[data['match_id'] == 9].drop(columns=[config.TARGET_COLUMN]) + with_fallback = small_model.predict(unseen) + pooled = np.clip(small_model.pooled_xgb.predict(small_model.feature_pipeline.transform(unseen)), 0, None) + np.testing.assert_allclose(with_fallback, pooled) + + +def test_true_revenue_share_is_one_at_the_optimum(data): + zones = data['seat_zone'].drop_duplicates().reset_index(drop=True) + base = zones.map(ZONE_BASE_PRICES).to_numpy() + np.testing.assert_allclose(true_revenue_share(1.25 * base, zones), 1.0, atol=1e-3) + + +def test_save_and_load_round_trip(tmp_path, data, small_model): + small_model.save(str(tmp_path)) + loaded = type(small_model).load(str(tmp_path)) + rows = data[data['match_id'] == 1].drop(columns=[config.TARGET_COLUMN]).head(20) + np.testing.assert_allclose(loaded.predict(rows), small_model.predict(rows)) + assert loaded.price_support == small_model.price_support