diff --git a/README.md b/README.md index d4df974..8567fe7 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ LLM routing methods are rapidly emerging, but existing implementations are often ORBIT standardizes the end-to-end routing workflow into a unified research stack: - **Unified pipeline and router interface**: dataset loading -> embedding extraction -> router training/inference -> standardized evaluation, all driven by consistent JSON configs. -- **Budget-aware evaluation**: sweep budgets to trace performance-cost trade-offs and report curve-level metrics such as **nAUC**, **Peak Score**, **QNC**, and **RCI**. +- **Budget-aware evaluation**: sweep budgets to trace performance-cost trade-offs and report curve-level metrics such as **nAUC**, **Peak Score**, and **QNC**. - **Extensible benchmark and method suite**: built-in support for unimodal and multimodal routing benchmarks, reproduced routing methods, and clean interfaces for adding datasets, embedding encoders, and routers. ORBIT currently supports large-scale routing evaluation with up to **8.65M** benchmark instances and is designed to accelerate reproducible research on budget-aware LLM routing systems. @@ -106,7 +106,7 @@ ORBIT reproduces **30 representative LLM routing methods** across training-free, ## Evaluation Metrics -ORBIT evaluates LLM routing under **budgeted inference** by sweeping budgets to obtain a **performance-cost trade-off curve**. It reports standardized metrics for fair comparison and diagnostic analysis. +ORBIT evaluates LLM routing under **budgeted inference** by sweeping budgets to obtain a **performance-cost trade-off curve**. It reports standardized metrics for comparing routing quality and cost-efficiency. ### Trade-off metrics @@ -114,10 +114,6 @@ ORBIT evaluates LLM routing under **budgeted inference** by sweeping budgets to - **Peak Score**: reports the best achievable utility within the evaluated budget range. Higher is better. - **QNC (Quality Neutral Cost)**: measures cost-efficiency under quality-neutral comparison. Lower is better. -### Diagnostic metrics - -- **RCI (Routing Collapse Index)**: diagnoses collapse and other failure modes beyond average utility. - ### Cost prediction protocol Every score-based learned router supplies a query-by-model cost matrix without using test labels. @@ -227,7 +223,7 @@ ORBIT includes a compact benchmark suite spanning text-only and multimodal routi > align candidate descriptions by name, reject invalid identities, and record coverage > and cost protocols. MMR-Bench V2 requires upstream image assets. Full R2-Bench is > approximately 25 GB; its costs are output-only references, not full bills, and -> Best Single/RCI are evaluated over model-budget actions. See [configuration, +> Best Single is evaluated over model-budget actions. See [configuration, > limitations and smoke instructions](docs/new_benchmarks.md) before running. --- diff --git a/docs/new_benchmarks.md b/docs/new_benchmarks.md index c01dc00..2d8088d 100644 --- a/docs/new_benchmarks.md +++ b/docs/new_benchmarks.md @@ -173,7 +173,7 @@ Qwen/Qwen3-0.6B@budget=100 All existing routers can learn/score these columns as usual. Choose an identical action pool for every compared method. This is **discrete action routing**; it neither implements R2-Router nor adds continuous-budget interpolation. -Best Single, RCI and the existing oracle are computed over candidate **actions**, +Best Single and the existing oracle are computed over candidate **actions**, not collapsed physical models. The sidecar maps every candidate to its physical model and nominal budget. @@ -240,7 +240,7 @@ and not an exhaustive run of all methods on all full datasets. `tests/test_all_methods_smoke.py` adds a CPU-only matrix covering every registered method on all three source-schema fixtures, plus all 24 compatible multimodal methods on MMR image fixtures. It calls real training, prediction, allocation, -RCI, nAUC and JSON output code; none of these are mocked. CI discovers this test +nAUC, Peak Score and JSON output code; none of these are mocked. CI discovers this test through the existing unittest entry point. For a persistent report with per-method logs and curves: @@ -259,7 +259,7 @@ python scripts/smoke_all_methods.py --real-data \ Report directories must be new or empty to preserve earlier evidence. The report records substitutions, sample/model counts, checkpoint hashes, nAUC and output -paths. Each method must produce a finite curve and log both nAUC and RCI. Pair +paths. Each method must produce a finite curve and log both nAUC and Peak Score. Pair routers' intentionally disabled `-inf` scores are accepted only outside the selected pair. Native cost regressions are checked before and after ORBIT's existing training-range clipping guard; negative raw values are recorded, not diff --git a/methods/CarrotRouter.py b/methods/CarrotRouter.py index f3a7689..050eb94 100644 --- a/methods/CarrotRouter.py +++ b/methods/CarrotRouter.py @@ -119,11 +119,8 @@ def _evaluate_predictions(self, perf_pred, cost_pred): rows = np.arange(len(perf_pred)) points = [] - quality_choices = None for mu in np.linspace(0.0, 1.0, self.mu_steps): choices = np.argmax(self.utility(perf_pred, cost_pred, mu), axis=1) - if quality_choices is None: - quality_choices = choices.copy() points.append( { "cost": float(np.mean(realized_cost[rows, choices])), @@ -131,5 +128,4 @@ def _evaluate_predictions(self, perf_pred, cost_pred): } ) - self.cal_rci(quality_choices, log_once=True) self.cal_metrics(points) diff --git a/methods/CausalRouter.py b/methods/CausalRouter.py index 0abf1a0..b50f7c6 100644 --- a/methods/CausalRouter.py +++ b/methods/CausalRouter.py @@ -180,17 +180,12 @@ def evaluate(self) -> None: lambda_pool = np.linspace(0.0, float(self.args["lambda_max"]), int(self.args["num_lambdas"])).astype(np.float32) - last_choice = None for lambda_val in lambda_pool: choice = self.predict(test_embs, lambda_val=lambda_val) # (N,) - last_choice = choice selected_perf = perf_mat[row_idx, choice] selected_costs = cost_mat[row_idx, choice] all_points.append({"cost": float(np.mean(selected_costs)), "performance": float(np.mean(selected_perf))}) - if last_choice is not None: - self.cal_rci(last_choice) - self.cal_metrics(all_points) # ----------------------------- diff --git a/methods/EARAMRouter.py b/methods/EARAMRouter.py index bebeafe..d9808c8 100644 --- a/methods/EARAMRouter.py +++ b/methods/EARAMRouter.py @@ -354,7 +354,6 @@ def _evaluate_predictions(self, perf_pred, cost_pred): rows = np.arange(len(probability)) points = [] - final_winners = None for value in self._value_grid(probability, cost_pred): winners, _, _ = self.auction_outcome(probability, cost_pred, value) active = winners >= 0 @@ -370,7 +369,5 @@ def _evaluate_predictions(self, perf_pred, cost_pred): "performance": float(selected_performance.mean()), } ) - final_winners = winners - self.cal_rci(final_winners, log_once=True) self.cal_metrics(points) diff --git a/methods/base.py b/methods/base.py index 891dd54..29c0aa1 100644 --- a/methods/base.py +++ b/methods/base.py @@ -122,10 +122,9 @@ def _evaluate_predictions(self, perf_pred, cost_pred): perf_mat = self.test_df[perf_cols_all].to_numpy(dtype=np.float32) # (N, M) cost_mat = self.test_df[cost_cols_all].to_numpy(dtype=np.float32) # (N, M) - all_points, best_idx = self._build_tradeoff_points( + all_points, _ = self._build_tradeoff_points( perf_pred, cost_pred, perf_mat, cost_mat ) - self.cal_rci(best_idx, log_once=True) self.cal_metrics(all_points) def _clip_predicted_costs(self, cost_pred): @@ -210,76 +209,6 @@ def cal_metrics(self,all_points): logging.info(f"[method.base.py] Saved Pareto frontier points to {json_path}\n") - def cal_rci(self, predict_idx, log_once: bool = True): - """ - RCI (0/1 per sample): - - 0 if chosen model is best AND not most expensive - - 0 if only the most expensive model(s) are best AND chosen is among them - - 1 otherwise - - Returns: - rci_mean: float, mean of per-sample rci in [0, 1] (lower is better) - rci_per_sample: np.ndarray shape (N,), values in {0,1} - """ - predict_idx = np.asarray(predict_idx, dtype=int) - N = int(predict_idx.shape[0]) - M = int(len(self.model_list)) - - if N == 0: - if log_once: - logging.info("[method.base.py] RCI: 0.0 (empty input)") - return 0.0, np.zeros((0,), dtype=np.int32) - - perf_cols = [f"model_{mid}_performance" for mid in range(M)] - cost_cols = [f"model_{mid}_cost" for mid in range(M)] - - # (N, M) - perf_mat = self.test_df[perf_cols].to_numpy() - cost_mat = self.test_df[cost_cols].to_numpy() - - rows = np.arange(N) - valid_choice = (predict_idx >= 0) & (predict_idx < M) - safe_predict_idx = np.clip(predict_idx, 0, max(M - 1, 0)) - - # Determine "most expensive" model(s) globally by average cost over test set. - # (Alternative: by max cost per-sample; but global is more stable.) - avg_costs = cost_mat.mean(axis=0) # (M,) - max_avg_cost = avg_costs.max() - most_expensive_mask = avg_costs == max_avg_cost # (M,) boolean - chosen_is_most_expensive = ( - most_expensive_mask[safe_predict_idx] & valid_choice - ) # (N,) - - # Best set per sample (ties allowed) - best_perf = perf_mat.max(axis=1) # (N,) - is_best = perf_mat == best_perf[:, None] # (N, M) - chosen_is_best = is_best[rows, safe_predict_idx] & valid_choice # (N,) - - # "Only most expensive is best" per sample: - # i.e., all best models are within the most-expensive set, and at least one best exists (always true). - best_is_subset_of_most_expensive = (is_best & (~most_expensive_mask[None, :])).sum(axis=1) == 0 # (N,) - - # Apply your rule: - # 0 if (chosen best and not most expensive) OR (only most expensive best and chosen best (=> chosen is expensive)) - ok_case_1 = chosen_is_best & (~chosen_is_most_expensive) - ok_case_2 = best_is_subset_of_most_expensive & chosen_is_best - ok = ok_case_1 | ok_case_2 - - rci_per_sample = (~ok).astype(np.int32) - rci_mean = float(rci_per_sample.mean()) - - if log_once: - logging.info( - "[method.base.py] RCI: %.6f | N=%d | ok(non-exp best)=%.4f | ok(only-exp-best)=%.4f | most_expensive_ids=%s", - rci_mean, - N, - float(ok_case_1.mean()), - float(ok_case_2.mean()), - np.where(most_expensive_mask)[0].tolist(), - ) - - return rci_mean, rci_per_sample - def _find_min_cost_for_target(self, pareto_points, target_accuracy): valid_points = [point for point in pareto_points if point["performance"] >= target_accuracy] diff --git a/scripts/smoke_all_methods.py b/scripts/smoke_all_methods.py index f6a22ce..b7288d9 100644 --- a/scripts/smoke_all_methods.py +++ b/scripts/smoke_all_methods.py @@ -255,8 +255,9 @@ def load_model(model, **kwargs): matches = re.findall(r"nAUC: ([0-9.eE+-]+)", text) if not matches or not all(0 <= float(value) <= 1 for value in matches): raise AssertionError("Missing/non-finite/out-of-range nAUC") - if not re.search(r"RCI: [0-9.]+", text): - raise AssertionError("Missing RCI evaluation") + peaks = re.findall(r"Maximum accuracy: ([0-9.eE+-]+)", text) + if not peaks or not all(np.isfinite(float(value)) for value in peaks): + raise AssertionError("Missing/non-finite Peak Score") result = {"benchmark": name, "method": method, "modality": args["modality"], "status": "PASS", "train": len(router.train_df), "test": len(router.test_df), "candidates": len(router.model_list), "nAUC": float(matches[-1]), "seconds": round(time.monotonic() - started, 3), diff --git a/scripts/smoke_new_benchmarks.py b/scripts/smoke_new_benchmarks.py index dd07950..7ae3246 100644 --- a/scripts/smoke_new_benchmarks.py +++ b/scripts/smoke_new_benchmarks.py @@ -128,7 +128,7 @@ def main(): expected = (len(router.test_df), len(router.model_list)) if performance.shape != expected or costs.shape != expected or not np.isfinite(performance).all() or not np.isfinite(costs).all(): raise AssertionError(f"Invalid smoke predictions: {name}/{method}") - router.evaluate() # actual shared nAUC/RCI/JSON path, unpatched + router.evaluate() # actual shared nAUC/JSON path, unpatched print(json.dumps({"benchmark": name, "method": method, "train": len(router.train_df), "test": len(router.test_df), "candidates": len(router.model_list), "status": "PASS", "representation": "smoke-hash text only"})) diff --git a/tests/test_all_methods_smoke.py b/tests/test_all_methods_smoke.py index dc948a9..7635072 100644 --- a/tests/test_all_methods_smoke.py +++ b/tests/test_all_methods_smoke.py @@ -1,7 +1,7 @@ """All-registered-method matrix with actual offline metric/result output. Large pretrained encoders are replaced at the input/checkpoint boundary only. -Router training, prediction, allocation, RCI, nAUC and JSON writing are real. +Router training, prediction, allocation, nAUC and JSON writing are real. """ import logging diff --git a/tests/test_paper_adapters.py b/tests/test_paper_adapters.py index b0de6ec..e27d10c 100644 --- a/tests/test_paper_adapters.py +++ b/tests/test_paper_adapters.py @@ -137,11 +137,8 @@ def test_profile_router_train_predict_smoke(self): texts=self.test_df["prompt"].tolist(), images=None ) performance, cost = router.predict(test_features) - with patch.object(router, "cal_rci") as rci, patch.object( - router, "cal_metrics" - ) as metrics: + with patch.object(router, "cal_metrics") as metrics: router.evaluate() - rci.assert_called_once() metrics.assert_called_once() self.assertEqual(performance.shape, (4, 3)) self.assertEqual(cost.shape, (4, 3)) @@ -162,11 +159,8 @@ def test_carrot_train_predict_smoke(self): texts=self.test_df["prompt"].tolist(), images=None ) performance, cost = router.predict(features) - with patch.object(router, "cal_rci") as rci, patch.object( - router, "cal_metrics" - ) as metrics: + with patch.object(router, "cal_metrics") as metrics: router.evaluate() - rci.assert_called_once() metrics.assert_called_once() self.assertEqual(performance.shape, (4, 3)) self.assertEqual(cost.shape, (4, 3)) @@ -199,11 +193,8 @@ def test_earam_train_predict_and_auction_smoke(self): evaluator_probability = router.predict_expost_acceptance( ["query a", "query b"], ["answer a", "answer b"] ) - with patch.object(router, "cal_rci") as rci, patch.object( - router, "cal_metrics" - ) as metrics: + with patch.object(router, "cal_metrics") as metrics: router.evaluate() - rci.assert_called_once() metrics.assert_called_once() self.assertEqual(probability.shape, (4, 3)) self.assertEqual(cost.shape, (4, 3)) @@ -267,11 +258,8 @@ def test_routefm_official_episode_adapter_smoke(self): texts=self.test_df["prompt"].tolist(), images=None ) performance, cost = router.predict(features) - with patch.object(router, "cal_rci") as rci, patch.object( - router, "cal_metrics" - ) as metrics: + with patch.object(router, "cal_metrics") as metrics: router.evaluate() - rci.assert_called_once() metrics.assert_called_once() self.assertEqual(router.context_query.shape, (3, 3, 4)) self.assertEqual(router.context_features.shape, (3, 3, 2)) @@ -303,11 +291,8 @@ def test_saverouter_sparse_supervision_adapter_smoke(self): texts=self.test_df["prompt"].tolist(), images=None ) performance, cost = router.predict(features) - with patch.object(router, "cal_rci") as rci, patch.object( - router, "cal_metrics" - ) as metrics: + with patch.object(router, "cal_metrics") as metrics: router.evaluate() - rci.assert_called_once() metrics.assert_called_once() self.assertEqual(router.supervision.n_observations, len(self.train_df) * 2) np.testing.assert_array_equal( diff --git a/tests/test_router_implementations.py b/tests/test_router_implementations.py index bd36ee8..7287961 100644 --- a/tests/test_router_implementations.py +++ b/tests/test_router_implementations.py @@ -1,10 +1,12 @@ import unittest +from unittest.mock import patch import numpy as np import torch from methods.Eagle import Eagle from methods.EmbedLLM import EmbedLLM +from methods.EARAMRouter import EARAMRouter from methods.ModelSAT import ModelSAT from methods.RouteLLM import PairConfig, _build_pairwise_pref, _resolve_pair_config from methods.UniRoute import UniRoute @@ -179,9 +181,11 @@ def test_native_cost_predictions_are_clipped_per_model(self): ) np.testing.assert_allclose(clipped, [[0.1, 0.6], [0.2, 0.8]]) - def test_rci_counts_null_auction_allocations_as_failures(self): - router = _ConcreteBase.__new__(_ConcreteBase) + def test_null_auction_allocations_have_zero_performance_and_cost(self): + router = EARAMRouter.__new__(EARAMRouter) router.model_list = ["cheap", "strong"] + router._training_cost_bounds = None + router.configured_values = [0.0, 1.0] import pandas as pd router.test_df = pd.DataFrame( @@ -192,9 +196,15 @@ def test_rci_counts_null_auction_allocations_as_failures(self): "model_1_cost": [0.9, 0.9], } ) - mean, per_sample = router.cal_rci(np.asarray([-1, 0]), log_once=False) - np.testing.assert_array_equal(per_sample, [1, 0]) - self.assertEqual(mean, 0.5) + probability = np.asarray([[0.0, 0.0], [0.9, 0.9]], dtype=np.float32) + cost = np.asarray([[0.1, 0.9], [0.1, 0.9]], dtype=np.float32) + with patch.object(router, "cal_metrics") as metrics: + router._evaluate_predictions(probability, cost) + metrics.assert_called_once() + points = metrics.call_args.args[0] + self.assertEqual(points[0], {"cost": 0.0, "performance": 0.0}) + self.assertAlmostEqual(points[1]["cost"], 0.05) + self.assertEqual(points[1]["performance"], 0.5) def test_modelsat_selects_last_non_padding_position(self): logits = torch.arange(2 * 4 * 3, dtype=torch.float32).reshape(2, 4, 3) diff --git a/utils/benchmark_adapters.py b/utils/benchmark_adapters.py index f4d5b54..11d3e28 100644 --- a/utils/benchmark_adapters.py +++ b/utils/benchmark_adapters.py @@ -536,7 +536,7 @@ def process(self, dataset): "cost_scope": "actual recorded output tokens times frozen reference output price; input and hidden reasoning costs unavailable" if mode == "output-usd" else "recorded output token count only; not a monetary cost", "price_sources": self.registry["price_sources"], "input_representation": "original_prompt only; nominal budget is an action attribute, not a query feature", - "candidate_semantics": "model-budget action; Best Single and RCI refer to fixed actions, not physical models", + "candidate_semantics": "model-budget action; Best Single refers to fixed actions, not physical models", "prompt_alignment": "key plus identical content after removing CR/LF only; prefer the released variant with most line breaks", "linebreak_variant_queries": len(linebreak_variants), "linebreak_variant_ids_sha256": hashlib.sha256("\n".join(sorted(linebreak_variants)).encode()).hexdigest(),