From e2b73137f93bf1bdd04fcff2a9a75b449c3cbdf9 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Fri, 16 Oct 2026 10:44:45 +0100 Subject: [PATCH 1/2] feat(proxy): report the mean of each request's saving on session and project rows A session or project row carried only totals, so its savings_percent was the overall figure: everything saved over everything that would have been sent, which a few large requests dominate. The mean of each request's own saving, where every request weighs the same, cannot be read back from totals. Each entry now keeps the sum of its requests' saving percentages and a count of the requests that contributed to it, and the row presents their quotient as avg_request_savings_percent beside savings_percent. The count is kept apart from requests because an entry persisted before the fields existed has requests the sum never saw; such an entry reports no mean until new requests accumulate rather than one understated for good. The accumulation and the row derivation, each written out twice for projects and sessions, are shared. --- headroom/proxy/savings_tracker.py | 137 ++++++++++++++++++-------- tests/test_request_savings_percent.py | 91 +++++++++++++++++ 2 files changed, 189 insertions(+), 39 deletions(-) create mode 100644 tests/test_request_savings_percent.py diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.py index 6c0d31352d..ab095b25e1 100644 --- a/headroom/proxy/savings_tracker.py +++ b/headroom/proxy/savings_tracker.py @@ -769,10 +769,76 @@ def _empty_project_entry() -> dict[str, Any]: "compression_savings_usd": 0.0, "total_input_tokens": 0, "total_input_cost_usd": 0.0, + # The mean of each request's own saving percentage needs the percentages themselves, which the totals above cannot give back: the sum of them, and how many requests contributed to it. The count is separate from ``requests`` because an entry persisted before these fields existed has requests the sum never saw, and dividing by ``requests`` would understate the mean for good. + "request_savings_percent_sum": 0.0, + "request_savings_percent_requests": 0, "last_activity_at": None, } +def _entry_view(entry: dict[str, Any]) -> dict[str, Any]: + """A per-project or per-session entry as the stats endpoints present it, with its two derived percentages. + + ``savings_percent`` is the overall figure: everything saved over everything that would have been sent, so a large request weighs more than a small one. ``avg_request_savings_percent`` is the mean of each request's own saving, every request weighing the same, and is ``None`` for an entry no request has contributed a percentage to (one persisted before the accumulators existed, until new requests arrive). + """ + view = dict(entry) + total_before = entry["tokens_saved"] + entry["total_input_tokens"] + view["savings_percent"] = round( + (entry["tokens_saved"] / total_before * 100) if total_before > 0 else 0.0, + 2, + ) + contributing = entry["request_savings_percent_requests"] + view["avg_request_savings_percent"] = ( + round(entry["request_savings_percent_sum"] / contributing, 2) if contributing > 0 else None + ) + return view + + +def _accumulate_entry( + entry: dict[str, Any], + *, + timestamp_dt: datetime, + requests_delta: int, + tokens_saved_delta: int, + savings_usd_delta: float, + input_tokens_delta: int, + input_cost_usd_delta: float, + request_savings_percent: float, +) -> None: + """Fold one request's deltas into a per-project or per-session entry. + + ``request_savings_percent`` is that request's own saving, in the same terms as the entry's overall figure (saved over saved plus input sent), added to a running sum with a count of how many requests contributed so the mean of the per-request figures can be read back. + """ + entry["requests"] += max(requests_delta, 0) + entry["tokens_saved"] += max(tokens_saved_delta, 0) + entry["compression_savings_usd"] = round( + entry["compression_savings_usd"] + savings_usd_delta, 6 + ) + entry["total_input_tokens"] += max(input_tokens_delta, 0) + entry["total_input_cost_usd"] = round( + entry["total_input_cost_usd"] + max(input_cost_usd_delta, 0.0), 6 + ) + if requests_delta > 0: + entry["request_savings_percent_sum"] = round( + entry["request_savings_percent_sum"] + request_savings_percent, 6 + ) + entry["request_savings_percent_requests"] += requests_delta + entry["last_activity_at"] = _to_utc_iso(timestamp_dt) + + +def _normalize_request_savings_percent(normalized: dict[str, Any], entry: dict[str, Any]) -> None: + """Carry a persisted entry's per-request percentage accumulators onto ``normalized``. + + Absent in state files written before the fields existed, which read as no requests contributing, so such an entry shows no mean until new requests accumulate rather than a mean over requests the sum never saw. + """ + normalized["request_savings_percent_sum"] = round( + _coerce_float(entry.get("request_savings_percent_sum")), 6 + ) + normalized["request_savings_percent_requests"] = _coerce_int( + entry.get("request_savings_percent_requests") + ) + + def _normalize_projects(raw: Any) -> dict[str, dict[str, Any]]: if not isinstance(raw, dict): return {} @@ -782,6 +848,7 @@ def _normalize_projects(raw: Any) -> dict[str, dict[str, Any]]: if cleaned_name is None or not isinstance(entry, dict): continue normalized = _empty_project_entry() + _normalize_request_savings_percent(normalized, entry) normalized["requests"] = _coerce_int(entry.get("requests")) normalized["tokens_saved"] = _coerce_int(entry.get("tokens_saved")) normalized["compression_savings_usd"] = round( @@ -825,6 +892,7 @@ def _normalize_sessions(raw: Any) -> dict[str, dict[str, Any]]: if key is None or not isinstance(entry, dict): continue normalized = _empty_session_entry() + _normalize_request_savings_percent(normalized, entry) normalized["requests"] = _coerce_int(entry.get("requests")) normalized["tokens_saved"] = _coerce_int(entry.get("tokens_saved")) normalized["compression_savings_usd"] = round( @@ -1154,6 +1222,11 @@ def record_request( delta_tokens_saved = _coerce_int(tokens_saved) delta_tool_tokens_saved = max(_coerce_int(tool_search_saved), 0) delta_input_tokens = _coerce_int(input_tokens) + # This request's own saving in the entries' terms (saved over saved plus input sent), so a mean of these and the entries' overall figure are measured the same way. + request_before = delta_tokens_saved + delta_input_tokens + request_savings_percent = ( + delta_tokens_saved / request_before * 100 if request_before > 0 else 0.0 + ) delta_output_tokens_saved = max(_coerce_int(output_tokens_saved), 0) delta_cache_read_tokens = _coerce_int(cache_read_tokens) priced = estimated_savings_usd @@ -1383,6 +1456,7 @@ def record_request( savings_usd_delta=delta_savings_usd, input_tokens_delta=delta_input_tokens, input_cost_usd_delta=delta_input_cost_usd, + request_savings_percent=request_savings_percent, ) self._record_session_locked( @@ -1393,6 +1467,7 @@ def record_request( savings_usd_delta=delta_savings_usd, input_tokens_delta=delta_input_tokens, input_cost_usd_delta=delta_input_cost_usd, + request_savings_percent=request_savings_percent, ) # In --mode cache, headroom's own compression (tokens_saved) is @@ -1540,6 +1615,7 @@ def _record_project_locked( savings_usd_delta: float = 0.0, input_tokens_delta: int = 0, input_cost_usd_delta: float = 0.0, + request_savings_percent: float = 0.0, ) -> None: """Accumulate per-project savings. Caller must hold ``self._lock``. @@ -1552,16 +1628,16 @@ def _record_project_locked( return projects: dict[str, dict[str, Any]] = self._state.setdefault("projects", {}) entry = projects.setdefault(name, _empty_project_entry()) - entry["requests"] += max(requests_delta, 0) - entry["tokens_saved"] += max(tokens_saved_delta, 0) - entry["compression_savings_usd"] = round( - entry["compression_savings_usd"] + savings_usd_delta, 6 + _accumulate_entry( + entry, + timestamp_dt=timestamp_dt, + requests_delta=requests_delta, + tokens_saved_delta=tokens_saved_delta, + savings_usd_delta=savings_usd_delta, + input_tokens_delta=input_tokens_delta, + input_cost_usd_delta=input_cost_usd_delta, + request_savings_percent=request_savings_percent, ) - entry["total_input_tokens"] += max(input_tokens_delta, 0) - entry["total_input_cost_usd"] = round( - entry["total_input_cost_usd"] + max(input_cost_usd_delta, 0.0), 6 - ) - entry["last_activity_at"] = _to_utc_iso(timestamp_dt) if len(projects) > DEFAULT_MAX_PROJECTS: evict = min( (key for key in projects if key != name), @@ -1582,6 +1658,7 @@ def _record_session_locked( savings_usd_delta: float = 0.0, input_tokens_delta: int = 0, input_cost_usd_delta: float = 0.0, + request_savings_percent: float = 0.0, ) -> None: """Accumulate per-session savings. Caller must hold ``self._lock``. @@ -1602,16 +1679,16 @@ def _record_session_locked( ]: del sessions[idle_key] entry = sessions.setdefault(key, _empty_session_entry()) - entry["requests"] += max(requests_delta, 0) - entry["tokens_saved"] += max(tokens_saved_delta, 0) - entry["compression_savings_usd"] = round( - entry["compression_savings_usd"] + savings_usd_delta, 6 - ) - entry["total_input_tokens"] += max(input_tokens_delta, 0) - entry["total_input_cost_usd"] = round( - entry["total_input_cost_usd"] + max(input_cost_usd_delta, 0.0), 6 + _accumulate_entry( + entry, + timestamp_dt=timestamp_dt, + requests_delta=requests_delta, + tokens_saved_delta=tokens_saved_delta, + savings_usd_delta=savings_usd_delta, + input_tokens_delta=input_tokens_delta, + input_cost_usd_delta=input_cost_usd_delta, + request_savings_percent=request_savings_percent, ) - entry["last_activity_at"] = _to_utc_iso(timestamp_dt) if len(sessions) > DEFAULT_MAX_SESSIONS: # Evict the least recently active session, not the smallest saver: sessions are a recency-scoped index (an operator asks "what did this run save"), so recency is the only honest eviction order. String compare on the ISO activity stamp is a time compare. evict = min( @@ -1660,13 +1737,7 @@ def _projects_snapshot_locked(self) -> dict[str, dict[str, Any]]: ) result: dict[str, dict[str, Any]] = {} for name, entry in ranked: - view = dict(entry) - total_before = entry["tokens_saved"] + entry["total_input_tokens"] - view["savings_percent"] = round( - (entry["tokens_saved"] / total_before * 100) if total_before > 0 else 0.0, - 2, - ) - result[name] = view + result[name] = _entry_view(entry) return result def _sessions_snapshot_locked(self) -> dict[str, dict[str, Any]]: @@ -1679,13 +1750,7 @@ def _sessions_snapshot_locked(self) -> dict[str, dict[str, Any]]: ) result: dict[str, dict[str, Any]] = {} for session_id, entry in ranked: - view = dict(entry) - total_before = entry["tokens_saved"] + entry["total_input_tokens"] - view["savings_percent"] = round( - (entry["tokens_saved"] / total_before * 100) if total_before > 0 else 0.0, - 2, - ) - result[session_id] = view + result[session_id] = _entry_view(entry) return result def session_response(self, session_id: str) -> dict[str, Any] | None: @@ -1700,13 +1765,7 @@ def session_response(self, session_id: str) -> dict[str, Any] | None: entry = self._state.get("sessions", {}).get(key) if not isinstance(entry, dict): return None - view = dict(entry) - total_before = entry["tokens_saved"] + entry["total_input_tokens"] - view["savings_percent"] = round( - (entry["tokens_saved"] / total_before * 100) if total_before > 0 else 0.0, - 2, - ) - return view + return _entry_view(entry) def _by_model_snapshot_locked(self) -> dict[str, dict[str, Any]]: """Per-model stats ranked by savings.""" diff --git a/tests/test_request_savings_percent.py b/tests/test_request_savings_percent.py new file mode 100644 index 0000000000..891cf3d245 --- /dev/null +++ b/tests/test_request_savings_percent.py @@ -0,0 +1,91 @@ +"""Tests for the mean of each request's own saving percentage on session and project rows.""" + +import json +from datetime import datetime, timezone + +from headroom.proxy.savings_tracker import SavingsTracker + +UTC = timezone.utc + +# Two requests whose own savings differ: one saves 10 of 100 (10%), the other 0 of 1000 (0%). The mean of the two is 5%, while the overall figure (10 saved of 1100 before) is far lower. +SMALL_SAVING = {"input_tokens": 90, "tokens_saved": 10} +LARGE_NO_SAVING = {"input_tokens": 1000, "tokens_saved": 0} + + +def _record_two(tracker: SavingsTracker) -> None: + tracker.record_request(model="gpt-4o", session_id="s", project="p", **SMALL_SAVING) + tracker.record_request(model="gpt-4o", session_id="s", project="p", **LARGE_NO_SAVING) + + +def test_session_row_carries_the_mean_of_per_request_percentages_beside_the_overall_one(tmp_path): + tracker = SavingsTracker(path=str(tmp_path / "savings.json")) + _record_two(tracker) + + row = tracker.session_response("s") + + assert row is not None + # Every request weighs the same: (10% + 0%) / 2. + assert row["avg_request_savings_percent"] == 5.0 + # The overall figure weighs by size: 10 saved over 10 + 90 + 0 + 1000 before. + assert row["savings_percent"] == 0.91 + + +def test_project_row_carries_the_same_mean_as_the_session_row(tmp_path): + tracker = SavingsTracker(path=str(tmp_path / "savings.json")) + _record_two(tracker) + + project_row = tracker.lifetime_response()["projects"]["p"] + + assert project_row["avg_request_savings_percent"] == 5.0 + + +def test_the_accumulators_survive_a_reload(tmp_path): + path = tmp_path / "savings.json" + _record_two(SavingsTracker(path=str(path))) + + reloaded = SavingsTracker(path=str(path)) + + row = reloaded.session_response("s") + assert row is not None + assert row["avg_request_savings_percent"] == 5.0 + # A request after the reload extends the same mean: (10% + 0% + 20%) / 3. + reloaded.record_request( + model="gpt-4o", session_id="s", project="p", input_tokens=80, tokens_saved=20 + ) + extended = reloaded.session_response("s") + assert extended is not None + assert extended["avg_request_savings_percent"] == 10.0 + + +def test_an_entry_persisted_before_the_accumulators_has_no_mean_until_requests_contribute(tmp_path): + path = tmp_path / "savings.json" + path.write_text( + json.dumps( + { + "schema_version": 6, + "lifetime": {}, + "display_session": None, + "history": [], + "sessions": { + "legacy": { + "requests": 500, + "tokens_saved": 5000, + "total_input_tokens": 100000, + "last_activity_at": datetime.now(UTC).isoformat(), + } + }, + } + ) + ) + tracker = SavingsTracker(path=str(path)) + + legacy = tracker.session_response("legacy") + assert legacy is not None + # The 500 earlier requests contributed no percentages, so there is no mean to report, rather than one dragged down by requests it never saw. + assert legacy["avg_request_savings_percent"] is None + + tracker.record_request(model="gpt-4o", session_id="legacy", input_tokens=90, tokens_saved=10) + resumed = tracker.session_response("legacy") + assert resumed is not None + # Only the one request that has contributed counts, not the 501 the entry has seen. + assert resumed["avg_request_savings_percent"] == 10.0 From 1909caf23df36fc03a7e9fef51322b820a9ea94f Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sun, 18 Oct 2026 14:30:28 +0100 Subject: [PATCH 2/2] docs(proxy): describe the per-request mean on the session and project rows --- docs/content/docs/proxy.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx index 32dbb89347..c2235c5c7e 100644 --- a/docs/content/docs/proxy.mdx +++ b/docs/content/docs/proxy.mdx @@ -452,7 +452,7 @@ curl "http://localhost:8787/stats-history?format=csv&series=weekly" ### `GET /stats/sessions/{session_id}` -Per-session savings for one harness session id, taken from the `x-claude-code-session-id` request header (or `x-headroom-session-id`). Same row shape as a per-project row from `/stats-lifetime`; loopback-only, like `/admin` and `/debug`. Requests without either header are counted globally but never attributed to a session. +Per-session savings for one harness session id, taken from the `x-claude-code-session-id` request header (or `x-headroom-session-id`). Same row shape as a per-project row from `/stats-lifetime`, which carries two percentages: `savings_percent`, everything saved over everything that would have been sent, so a large request weighs more than a small one, and `avg_request_savings_percent`, the mean of each request's own saving with every request weighing the same (`null` for a row whose requests predate the field, until new ones arrive). Loopback-only, like `/admin` and `/debug`. Requests without either header are counted globally but never attributed to a session. ```bash curl http://localhost:8787/stats/sessions/0f1e2d3c-4b5a-6978-8796-a5b4c3d2e1f0