From 44acd6a3a26c052c77e98b67b78fcfadfda95e2b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:18:25 +0900 Subject: [PATCH 1/2] experiment: trim psychometric rows without full scans --- contextual_orchestrator/psychometric_routing.py | 11 ++++------- scripts/benchmark_psychometric_routing.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/contextual_orchestrator/psychometric_routing.py b/contextual_orchestrator/psychometric_routing.py index bef8645fc..7ab731647 100644 --- a/contextual_orchestrator/psychometric_routing.py +++ b/contextual_orchestrator/psychometric_routing.py @@ -61,13 +61,10 @@ def observe_context_id( values = (int(accepted), *(int(value) for value in irt_row)) if any(value not in (0, 1) for value in values): raise ValueError("judge IRT rows must be dichotomous") - stale = [ - key - for key in self._responses - if key[:2] == (agent_id, context_id) and key[2] >= len(values) - ] - for key in stale: - del self._responses[key] + stale_index = len(values) + while (agent_id, context_id, stale_index) in self._responses: + del self._responses[(agent_id, context_id, stale_index)] + stale_index += 1 for item_index, value in enumerate(values): self._responses[(agent_id, context_id, item_index)] = value while len(self._contexts) > self.max_contexts: diff --git a/scripts/benchmark_psychometric_routing.py b/scripts/benchmark_psychometric_routing.py index c614b06f2..c89f68fc7 100644 --- a/scripts/benchmark_psychometric_routing.py +++ b/scripts/benchmark_psychometric_routing.py @@ -56,6 +56,19 @@ def main() -> None: samples_ms.append((time.perf_counter_ns() - started_ns) / 1_000_000) assert len(ranked) == len(model_ids) + observation_samples_ms: list[float] = [] + for sample_index in range(101): + started_ns = time.perf_counter_ns() + evidence.observe( + "context_511", + "model_3", + bool(sample_index % 2), + None, + irt_row=(int(not sample_index % 2),), + ) + observation_samples_ms.append( + (time.perf_counter_ns() - started_ns) / 1_000_000 + ) print( json.dumps( { @@ -63,6 +76,8 @@ def main() -> None: "models": len(model_ids), "items_per_context": 2, "median_fit_and_rank_ms": statistics.median(samples_ms), + "median_observe_ms": statistics.median(observation_samples_ms), + "p95_observe_ms": sorted(observation_samples_ms)[95], "samples_ms": samples_ms, }, sort_keys=True, From d213489248580b54f1e92f8a05b97e82aa5c5ae2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:19:41 +0900 Subject: [PATCH 2/2] docs(routing): record observation latency KPI --- CHANGELOG.md | 3 +++ docs/doctoring/measured-routing-evidence.md | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0b268c9..42c12294b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- Psychometric observation replacement now removes only the existing row's + contiguous trailing items instead of scanning the complete response ledger; + the checked-in benchmark reports fit/rank and observation p50/p95 separately. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. diff --git a/docs/doctoring/measured-routing-evidence.md b/docs/doctoring/measured-routing-evidence.md index ca24b1532..0141f669e 100644 --- a/docs/doctoring/measured-routing-evidence.md +++ b/docs/doctoring/measured-routing-evidence.md @@ -53,6 +53,12 @@ latency or answer accuracy. The next accuracy experiment must use a held-out model-query matrix and report log loss or Brier score alongside routing regret; true-parameter simulations must continue to report RMSE. +The successor observation-path experiment uses the same 512-context ledger. +Replacing one model/context row fell from p50 0.133833 ms and p95 0.152166 ms +on `b2f90116` to p50 0.000875 ms and p95 0.001000 ms. The benchmark now emits +both fields. This is local gateway bookkeeping evidence; the fit, held-out +quality, and provider latency remain separate KPIs. + ## APA 7 references Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large