Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions scripts/demo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,23 @@ banner() {
echo "============================================================"
}

# The artifact sections below read committed results/* files. Those are no
# longer tracked, so each section is guarded: a missing artifact prints a
# skip line instead of aborting the demo under `set -e`.
have() {
if [ -f "$1" ]; then return 0; fi
echo " (skipped -- $1 not present; regenerate it with the pipeline)"
return 1
}

banner "Day 1 -- temporal-split fix is committed"
echo "Pre-fix (random split, leaked future txns):"
echo " train_test_split(X, y, test_size=0.2, stratify=y)"
echo "Post-fix (per-source temporal split):"
grep -n "def temporal_split_per_source" src/train.py | head -1
echo ""
echo "Honest sparkov_test AUC (from results/baseline_metrics.json):"
if have results/baseline_metrics.json; then
python - <<'PY'
import json
b = json.load(open("results/baseline_metrics.json"))
Expand All @@ -47,20 +57,24 @@ auc = held.get("our_auc")
ag = held.get("baselines", {}).get("AutoGluon", {})
print(f" sparkov_test_auc = {auc} (vs AutoGluon {ag.get('baseline_auc', 'n/a')}, delta {ag.get('delta', 'n/a')})")
PY
fi

banner "Day 2 -- Pandas == Dask determinism (1K rows)"
python -m pytest tests/test_features_determinism.py -q --disable-warnings 2>&1 | tail -3

banner "Day 2 -- MLflow registry rollback latency (cached results)"
if have results/registry_rollback_times.csv; then
python - <<'PY'
import pandas as pd
df = pd.read_csv("results/registry_rollback_times.csv")
print(df[["iteration", "alias_flip_seconds", "audit_tag_seconds", "total_seconds"]].to_string(index=False))
print(f"\nMedian alias-flip: {df['alias_flip_seconds'].median() * 1000:.1f} ms")
print(f"Max alias-flip: {df['alias_flip_seconds'].max() * 1000:.1f} ms")
PY
fi

banner "Day 3 -- 30-day synthetic drift replay (cached results)"
if have results/drift_replay_summary.json; then
python - <<'PY'
import json, pandas as pd
summary = json.load(open("results/drift_replay_summary.json"))
Expand All @@ -71,33 +85,32 @@ print(f"Detected fires: {summary.get('detected_fire_days', '<see per_day.csv>')
print(f"Precision: {summary.get('precision', 'n/a')}")
print(f"Recall: {summary.get('recall', 'n/a')}")
PY
fi

banner "Day 3 -- auto-retrain events"
if have results/drift_retrain_events.csv; then
python - <<'PY'
import pandas as pd
df = pd.read_csv("results/drift_retrain_events.csv")
cols = ["triggered_on_day", "shadow_auprc", "prod_auprc", "promote_decision",
"new_model_version", "seconds_end_to_end"]
print(df[cols].to_string(index=False))
PY
fi

banner "Day 6 -- Sentinel champion vs naive notebook vs Claude Opus 4.6"
if have results/day06/frontier_comparison.csv; then
python - <<'PY'
import pandas as pd
df = pd.read_csv("results/day06/frontier_comparison.csv")
df_disp = df[["strategy", "auc", "auprc", "f1_at_0_5", "latency_s_per_query", "cost_usd_at_1k_qps_per_day"]]
df_disp.columns = ["strategy", "AUC", "AUPRC", "F1@0.5", "latency/q (s)", "$/day @ 1k qps"]
print(df_disp.to_string(index=False))
PY
fi

banner "Sprint complete -- 7 days, 31 tests passing, AutoGluon gap closed"
echo "Sources:"
echo " results/baseline_metrics.json (Day 1 honest AUC)"
echo " results/throughput_speedup.csv (Day 2 Pandas vs Dask)"
echo " results/registry_rollback_times.csv (Day 2 alias-flip)"
echo " results/drift_replay_summary.json (Day 3 30-day replay)"
echo " results/drift_retrain_events.csv (Day 3 auto-retrain)"
echo " results/day05/ (Day 5 sweep + fix)"
echo " results/day06/ (Day 6 frontier + ablation)"
echo "Artifacts are regenerated into results/ by the pipeline; none are"
echo "tracked in the repo. Run 'dvc repro' and the src/ harnesses to rebuild."
echo ""
echo "Open the dashboard: streamlit run app.py -> sidebar 'Ops'"
Loading