diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0b268c9..42c12294b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- Psychometric observation replacement now removes only the existing row's + contiguous trailing items instead of scanning the complete response ledger; + the checked-in benchmark reports fit/rank and observation p50/p95 separately. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. diff --git a/contextual_orchestrator/psychometric_routing.py b/contextual_orchestrator/psychometric_routing.py index bef8645fc..7ab731647 100644 --- a/contextual_orchestrator/psychometric_routing.py +++ b/contextual_orchestrator/psychometric_routing.py @@ -61,13 +61,10 @@ def observe_context_id( values = (int(accepted), *(int(value) for value in irt_row)) if any(value not in (0, 1) for value in values): raise ValueError("judge IRT rows must be dichotomous") - stale = [ - key - for key in self._responses - if key[:2] == (agent_id, context_id) and key[2] >= len(values) - ] - for key in stale: - del self._responses[key] + stale_index = len(values) + while (agent_id, context_id, stale_index) in self._responses: + del self._responses[(agent_id, context_id, stale_index)] + stale_index += 1 for item_index, value in enumerate(values): self._responses[(agent_id, context_id, item_index)] = value while len(self._contexts) > self.max_contexts: diff --git a/docs/doctoring/measured-routing-evidence.md b/docs/doctoring/measured-routing-evidence.md index db6a91ed1..b44669d29 100644 --- a/docs/doctoring/measured-routing-evidence.md +++ b/docs/doctoring/measured-routing-evidence.md @@ -57,6 +57,12 @@ latency or answer accuracy. The next accuracy experiment must use a held-out model-query matrix and report log loss or Brier score alongside routing regret; true-parameter simulations must continue to report RMSE. +The successor observation-path experiment uses the same 512-context ledger. +Replacing one model/context row fell from p50 0.133833 ms and p95 0.152166 ms +on `b2f90116` to p50 0.000875 ms and p95 0.001000 ms. The benchmark now emits +both fields. This is local gateway bookkeeping evidence; the fit, held-out +quality, and provider latency remain separate KPIs. + ## APA 7 references Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large diff --git a/scripts/benchmark_psychometric_routing.py b/scripts/benchmark_psychometric_routing.py index 4b16fb7ef..1fcc0af36 100644 --- a/scripts/benchmark_psychometric_routing.py +++ b/scripts/benchmark_psychometric_routing.py @@ -65,6 +65,19 @@ def main() -> None: samples_ms.append((time.perf_counter_ns() - started_ns) / 1_000_000) assert len(ranked) == len(model_ids) + observation_samples_ms: list[float] = [] + for sample_index in range(101): + started_ns = time.perf_counter_ns() + evidence.observe( + "context_511", + "model_3", + bool(sample_index % 2), + None, + irt_row=(int(not sample_index % 2),), + ) + observation_samples_ms.append( + (time.perf_counter_ns() - started_ns) / 1_000_000 + ) print( json.dumps( { @@ -72,6 +85,8 @@ def main() -> None: "models": len(model_ids), "items_per_context": 2, "median_fit_and_rank_ms": statistics.median(samples_ms), + "median_observe_ms": statistics.median(observation_samples_ms), + "p95_observe_ms": sorted(observation_samples_ms)[95], "samples_ms": samples_ms, }, sort_keys=True,