From 5ad1f493782791ccca1828fe415c458ba69e7ff2 Mon Sep 17 00:00:00 2001 From: AIGNLAI Date: Thu, 15 Oct 2026 14:55:37 +0800 Subject: [PATCH] Add VL-RouterBench and multimodal RouteFM/ProfileRouter adapters --- README.md | 11 +- configs/benchmarks/vlrouterbench.json | 32 ++ configs/datasets/vlrouterbench.json | 92 +++++ configs/description/tasks/VLRouterBench.json | 16 + configs/description/vlrouterbench.json | 121 ++++++ configs/examples/profile_multimodal.json | 3 + configs/examples/routefm_qwen.json | 23 ++ configs/routers/ProfileRouter.json | 9 +- docs/multimodal_adapters.md | 160 ++++++++ docs/new_benchmarks.md | 9 +- docs/vl_routerbench.md | 202 +++++++++ main.py | 5 + methods/ProfileRouter.py | 14 + methods/RouteFM.py | 19 +- requirements-ci.txt | 1 + scripts/download_vlrouterbench.py | 57 +++ scripts/smoke_all_methods.py | 82 +++- tests/test_all_methods_smoke.py | 12 +- tests/test_multimodal_adapters.py | 221 ++++++++++ tests/test_router_implementations.py | 12 +- tests/test_vl_routerbench.py | 317 +++++++++++++++ utils/data.py | 6 + utils/embedding.py | 68 +++- utils/qwen_vl_embedding.py | 140 +++++++ utils/vl_routerbench.py | 407 +++++++++++++++++++ 25 files changed, 2003 insertions(+), 36 deletions(-) create mode 100644 configs/benchmarks/vlrouterbench.json create mode 100644 configs/datasets/vlrouterbench.json create mode 100644 configs/description/tasks/VLRouterBench.json create mode 100644 configs/description/vlrouterbench.json create mode 100644 configs/examples/profile_multimodal.json create mode 100644 configs/examples/routefm_qwen.json create mode 100644 docs/multimodal_adapters.md create mode 100644 docs/vl_routerbench.md create mode 100644 scripts/download_vlrouterbench.py create mode 100644 tests/test_multimodal_adapters.py create mode 100644 tests/test_vl_routerbench.py create mode 100644 utils/qwen_vl_embedding.py create mode 100644 utils/vl_routerbench.py diff --git a/README.md b/README.md index 8778f5f..030f005 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Useful links: ## What's New +- **2026-10**: Added **VL-RouterBench** support for vision-language model routing through the existing ORBIT pipeline. The adapter preserves named outcomes and prepared splits, resolves TSV image assets, and records estimated-token reference costs. See [preparation and protocol](docs/vl_routerbench.md). - **2026-10**: Added adapters for **xRouteBench**, **MMR-Bench V2**, and **R2-Bench**, allowing existing routers to evaluate newer released outcomes through the same ORBIT pipeline. R2's model-budget pairs are ordinary routing candidates; release-specific cost and input limitations are documented in [the adapter guide](docs/new_benchmarks.md). - **2026-10**: Added **RouteFM**, the frozen in-context foundation router that transfers to new candidate pools through behavioral context, and **SAVERouter**, which adaptively acquires fixed-K sparse feedback and accounts for supervision expenditure. [[RouteFM Paper]](https://arxiv.org/abs/2609.37362) [[RouteFM Code]](https://github.com/LAMDA-Model-Reuse/RouteFM) [[SAVERouter Paper]](https://arxiv.org/abs/2609.37402) [[SAVERouter Code]](https://github.com/LAMDA-Model-Reuse/SaveRouter) - **2026-08**: ORBIT has been accepted by the *Frontiers of Computer Science* (FCS) special column **[Code & Data](https://journal.hep.com.cn/fcs/EN/subject/showCollection.do?subjectId=1710741206314)**. [[Paper]](https://doi.org/10.1007/s11704-026-61310-5) @@ -105,10 +106,10 @@ ORBIT reproduces **30 representative LLM routing methods** across training-free, - **TRouter**: Learns task-aware query and model representations for performance-cost routing. [[Paper]](https://arxiv.org/abs/2604.09377) - **UniRoute**: Represents model capabilities through cluster-level prediction errors and learns a query-to-cluster router. [[Paper]](https://openreview.net/forum?id=ka82fvJ5f1) - **InferenceDynamics**: Builds parameter-free model indexes from ranked capability and knowledge profiles for structured model-query matching. [[Paper]](https://arxiv.org/abs/2505.16303) -- **ProfileRouter**: Implements RouteProfile's training-free Emb-GNN profile construction and cosine-similarity SimRouter using public model descriptions, family metadata, and optional reported benchmark scores. [[Paper]](https://arxiv.org/abs/2605.00180) +- **ProfileRouter**: Implements RouteProfile's training-free Emb-GNN profile construction and cosine-similarity SimRouter using public metadata. Also supports an explicitly labeled CLIP-aligned multimodal extension. [[Paper]](https://arxiv.org/abs/2605.00180) [[Multimodal adapter]](docs/multimodal_adapters.md) - **CarrotRouter**: Uses CARROT's KNN plug-in estimators for query-level performance and cost, then applies its original cost-aware rate-optimal scalarization. [[Paper]](https://arxiv.org/abs/2502.03261) - **EARAMRouter**: Trains independent provider-side success predictors and allocates queries through EA-RAM's largest-positive-surplus reverse auction with runner-up externality payments. [[Paper]](https://arxiv.org/abs/2608.12719) -- **RouteFM**: Uses the official frozen foundation router and anonymous per-candidate behavioral context to predict target-query quality and relative cost without target-domain parameter updates. ORBIT defaults to the released BGE text checkpoint. [[Paper]](https://arxiv.org/abs/2609.37362) [[Code]](https://github.com/LAMDA-Model-Reuse/RouteFM) +- **RouteFM**: Uses the official frozen foundation router and anonymous per-candidate behavioral context to predict target-query quality and relative cost without target-domain parameter updates. Defaults to BGE text routing; Qwen multimodal routing uses joint Qwen3-VL-Embedding-8B features. [[Paper]](https://arxiv.org/abs/2609.37362) [[Code]](https://github.com/LAMDA-Model-Reuse/RouteFM) [[Configuration]](docs/multimodal_adapters.md) - **SAVERouter**: Adaptively reveals exactly K model outcomes per training query, then combines hierarchical group-model shrinkage with query-level residual refinement and native sparse cost estimates. [[Paper]](https://arxiv.org/abs/2609.37402) [[Code]](https://github.com/LAMDA-Model-Reuse/SaveRouter) --- @@ -196,6 +197,7 @@ python main.py --dataset Routerbench --method SaveRouter python main.py --dataset XRouteBench --method knn python main.py --dataset MMRBenchV2 --method GraphRouter python main.py --dataset R2Bench --method CarrotRouter +python main.py --dataset VLRouterBench --method GraphRouter ``` Dataset and method names are matched case-insensitively. Device selection defaults to @@ -223,6 +225,11 @@ ORBIT includes a compact benchmark suite spanning text-only and multimodal routi | **xRouteBench** | Text / text-view | 18 | Official train/test outcomes with recorded token usage and frozen pricing. | | **MMR-Bench V2** | Multimodal | 44 | Versioned 18-task release; common successful support and output-only reference costs. | | **R2-Bench** | Text | 157 actions | 10 physical models with released nominal output-budget settings. | +| **VL-RouterBench** | Multimodal | 17 | 14-task upstream exports; train/dev/test IDs and explicit estimated-token reference costs. | + +> VL-RouterBench requires one-time preparation of its raw archive with upstream +> Steps 1 and 2. ORBIT reads named JSONL/Parquet/CSV exports, not positional or +> pickle matrices. See [data preparation, image handling and evaluation limitations](docs/vl_routerbench.md). > Note: RouterEval provides 12 dataset files with nested model pools. To ensure a consistent evaluation protocol, ORBIT deterministically sorts the common models from Group A's six files and aligns performance and cost columns to that canonical order. The bundled metadata currently contains 239 model descriptions; description-based routers validate coverage by model name and report any missing entries instead of silently misaligning them. diff --git a/configs/benchmarks/vlrouterbench.json b/configs/benchmarks/vlrouterbench.json new file mode 100644 index 0000000..f86555a --- /dev/null +++ b/configs/benchmarks/vlrouterbench.json @@ -0,0 +1,32 @@ +{ + "_comment": "Prepared upstream Steps 1 and 2; see docs/vl_routerbench.md. No pickle/matrix loading.", + "modality": "text+image", + "voting": "weighted", + "seed": 42, + "description_path": "./configs/description/vlrouterbench.json", + "device": "auto", + "dataset": { + "name": "VLRouterBench", + "dataset_dir": "./data/vl_routerbench", + "revision": "6e44fbf50e170f8e72734b49f63f2b52a8866377", + "cost_scaling": "train-max", + "local_files_only": true, + "missing_policy": "error", + "text_overlap_policy": "error", + "estimated_output_policy": "allow", + "include_responses": false, + "multi_image": "contact-sheet", + "image_cell_size": 512, + "split": { + "mode": "official" + } + }, + "embeddings": { + "text_model": "ViT-B/16", + "image_model": "ViT-B/16", + "batch_size": 128, + "normalize": true, + "training": false, + "out_dim": 1024 + } +} diff --git a/configs/datasets/vlrouterbench.json b/configs/datasets/vlrouterbench.json new file mode 100644 index 0000000..c80cfcc --- /dev/null +++ b/configs/datasets/vlrouterbench.json @@ -0,0 +1,92 @@ +{ + "hf_id": "KinghtH/VL-RouterBench", + "revision": "6e44fbf50e170f8e72734b49f63f2b52a8866377", + "upstream_revision": "f5a41ce83d6efe67489b35bb7a07ae72976a9aba", + "pricing_source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/pricing.yaml", + "models": { + "GPT4o": { + "input_price_per_1m": 2.5, + "output_price_per_1m": 10 + }, + "GeminiFlash2-5": { + "input_price_per_1m": 0.3, + "output_price_per_1m": 2.4 + }, + "Gemma3-27B": { + "input_price_per_1m": 0.35, + "output_price_per_1m": 0.5 + }, + "InternVL2_5-78B": { + "input_price_per_1m": 1, + "output_price_per_1m": 1.5 + }, + "Janus-Pro-1B": { + "input_price_per_1m": 0.05, + "output_price_per_1m": 0.05 + }, + "Janus-Pro-7B": { + "input_price_per_1m": 0.18, + "output_price_per_1m": 0.25 + }, + "Kimi-VL-A3B-Thinking-2506": { + "input_price_per_1m": 0.2, + "output_price_per_1m": 0.25 + }, + "MiMo-VL-7B-RL": { + "input_price_per_1m": 0.2, + "output_price_per_1m": 0.3 + }, + "Phi-3.5-Vision": { + "input_price_per_1m": 0.1, + "output_price_per_1m": 0.1 + }, + "Pixtral-12B": { + "input_price_per_1m": 0.25, + "output_price_per_1m": 0.35 + }, + "Qianfan-VL-8B": { + "input_price_per_1m": 0.18, + "output_price_per_1m": 0.25 + }, + "Qwen2.5-VL-32B-Instruct": { + "input_price_per_1m": 0.4, + "output_price_per_1m": 0.6 + }, + "Qwen2.5-VL-72B-Instruct": { + "input_price_per_1m": 0.8, + "output_price_per_1m": 1.2 + }, + "SmolVLM2": { + "input_price_per_1m": 0.06, + "output_price_per_1m": 0.06 + }, + "deepseek_vl2": { + "input_price_per_1m": 0.35, + "output_price_per_1m": 0.5 + }, + "deepseek_vl2_tiny": { + "input_price_per_1m": 0.05, + "output_price_per_1m": 0.05 + }, + "llava_next_vicuna_7b": { + "input_price_per_1m": 0.2, + "output_price_per_1m": 0.2 + } + }, + "benchmarks": { + "MMMU_DEV_VAL": "vqa_mc", + "MathVista_MINI": "math", + "MathVision_MINI": "math", + "MathVerse_MINI": "math", + "MMBench_DEV_EN_V11": "vqa_mc", + "RealWorldQA": "vqa_oe", + "MMStar": "vqa_mc", + "HallusionBench": "vqa_mc", + "TextVQA_VAL": "vqa_oe", + "ChartQA_TEST": "ocr_qa", + "DocVQA_VAL": "ocr_qa", + "InfoVQA_VAL": "ocr_qa", + "AI2D_TEST": "vqa_mc", + "OCRBench": "ocr_qa" + } +} diff --git a/configs/description/tasks/VLRouterBench.json b/configs/description/tasks/VLRouterBench.json new file mode 100644 index 0000000..5bb734a --- /dev/null +++ b/configs/description/tasks/VLRouterBench.json @@ -0,0 +1,16 @@ +{ + "MMMU_DEV_VAL": "Multimodal multi-task understanding (dev + validation)", + "MathVista_MINI": "Math visual reasoning and understanding (MINI version)", + "MathVision_MINI": "Math visual reasoning (MINI version)", + "MathVerse_MINI": "Math reasoning (MINI version)", + "MMBench_DEV_EN_V11": "Multimodal benchmark (dev set, English V11)", + "RealWorldQA": "Real-world question answering", + "MMStar": "MMStar multimodal evaluation", + "HallusionBench": "Hallucination detection benchmark", + "TextVQA_VAL": "TextVQA (validation set)", + "ChartQA_TEST": "Chart question answering (test set)", + "DocVQA_VAL": "Document question answering (validation set)", + "InfoVQA_VAL": "InfoVQA (validation set)", + "AI2D_TEST": "AI2D diagram understanding (test set)", + "OCRBench": "OCR capability benchmark" +} diff --git a/configs/description/vlrouterbench.json b/configs/description/vlrouterbench.json new file mode 100644 index 0000000..0ccc34d --- /dev/null +++ b/configs/description/vlrouterbench.json @@ -0,0 +1,121 @@ +[ + { + "id": 0, + "model_name": "GPT4o", + "family": "GPT", + "description": "GPT-4o OpenAI flagship multimodal model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 1, + "model_name": "GeminiFlash2-5", + "family": "Gemini", + "description": "Gemini Flash 2.5 Google efficient multimodal model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 2, + "model_name": "Gemma3-27B", + "family": "Gemma", + "description": "Gemma3-27B vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 3, + "model_name": "InternVL2_5-78B", + "family": "InternVL", + "description": "InternVL2.5-78B multimodal model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 4, + "model_name": "Janus-Pro-1B", + "family": "Janus", + "description": "Janus-Pro-1B lightweight vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 5, + "model_name": "Janus-Pro-7B", + "family": "Janus", + "description": "Janus-Pro-7B vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 6, + "model_name": "Kimi-VL-A3B-Thinking-2506", + "family": "Kimi", + "description": "Kimi-VL-A3B-Thinking-2506 thinking-oriented vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 7, + "model_name": "MiMo-VL-7B-RL", + "family": "MiMo", + "description": "MiMo-VL-7B-RL vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 8, + "model_name": "Phi-3.5-Vision", + "family": "Phi", + "description": "Phi-3.5-Vision multimodal model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 9, + "model_name": "Pixtral-12B", + "family": "Pixtral", + "description": "Pixtral-12B vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 10, + "model_name": "Qianfan-VL-8B", + "family": "Qianfan", + "description": "Qianfan-VL-8B vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 11, + "model_name": "Qwen2.5-VL-32B-Instruct", + "family": "Qwen", + "description": "Qwen2.5-VL-32B-Instruct vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 12, + "model_name": "Qwen2.5-VL-72B-Instruct", + "family": "Qwen", + "description": "Qwen2.5-VL-72B-Instruct vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 13, + "model_name": "SmolVLM2", + "family": "SmolVLM", + "description": "SmolVLM2 lightweight vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 14, + "model_name": "deepseek_vl2", + "family": "DeepSeek", + "description": "DeepSeek-VL2 vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 15, + "model_name": "deepseek_vl2_tiny", + "family": "DeepSeek", + "description": "DeepSeek-VL2-Tiny lightweight vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + }, + { + "id": 16, + "model_name": "llava_next_vicuna_7b", + "family": "LLaVA", + "description": "LLaVA-Next-Vicuna-7B vision-language model. A released VL-RouterBench vision-language candidate; this historical identifier is not assumed to be a current API endpoint.", + "source": "https://github.com/K1nght/VL-RouterBench/blob/f5a41ce83d6efe67489b35bb7a07ae72976a9aba/config/models.yaml" + } +] diff --git a/configs/examples/profile_multimodal.json b/configs/examples/profile_multimodal.json new file mode 100644 index 0000000..b6e645f --- /dev/null +++ b/configs/examples/profile_multimodal.json @@ -0,0 +1,3 @@ +{ + "modality": "text+image" +} diff --git a/configs/examples/routefm_qwen.json b/configs/examples/routefm_qwen.json new file mode 100644 index 0000000..8e4ed41 --- /dev/null +++ b/configs/examples/routefm_qwen.json @@ -0,0 +1,23 @@ +{ + "modality": "text+image", + "embeddings": { + "text_model": null, + "image_model": null, + "joint_model": "Qwen/Qwen3-VL-Embedding-8B", + "out_dim": 4096, + "normalize": true, + "training": false, + "batch_size": 2, + "qwen": { + "backend": "local", + "model_dir": "./Qwen3-VL-Embedding-8B" + } + }, + "routefm": { + "encoder": "qwen", + "checkpoint": null, + "cache_dir": null, + "context_size": 8, + "target_batch_size": 128 + } +} diff --git a/configs/routers/ProfileRouter.json b/configs/routers/ProfileRouter.json index c58896f..5fed20c 100644 --- a/configs/routers/ProfileRouter.json +++ b/configs/routers/ProfileRouter.json @@ -3,7 +3,14 @@ "modality": "text", "profile": { "aggregation_hops": 4, - "normalization": "sym" + "normalization": "sym", + "multimodal_embeddings": { + "text_model": "ViT-B/16", + "image_model": "ViT-B/16", + "out_dim": 512, + "batch_size": 8, + "text_weight": 0.5 + } }, "embeddings": { "text_model": "allenai/longformer-base-4096", diff --git a/docs/multimodal_adapters.md b/docs/multimodal_adapters.md new file mode 100644 index 0000000..9dafddd --- /dev/null +++ b/docs/multimodal_adapters.md @@ -0,0 +1,160 @@ +# RouteFM-Qwen and ProfileRouter multimodal adapters + +The dataset, `train`/`predict`/`evaluate` interfaces and metric definitions remain +unchanged. `Embedder.run_embed(texts=..., images=...)` now additionally supports +an explicitly selected joint encoder and aligned weighted fusion. Existing +methods still concatenate their independent feature vectors by default. + +## RouteFM-Qwen + +Text-only runs still default to the released RouteFM-BGE checkpoint. To use +the released Qwen router on a multimodal benchmark: + +```bash +python main.py --dataset VLRouterBench --method RouteFM \ + --config configs/examples/routefm_qwen.json --device cuda:0 +``` + +`--config` applies **top-level replacements** after benchmark/router defaults; +it does not recursively merge nested objects. `--device` takes final precedence. +Prepare the benchmark's upstream exports as described in +[VL-RouterBench](vl_routerbench.md) and configure its dataset directory first. +Set `routefm.checkpoint` to a local official Qwen router checkpoint, or leave it +null to use the pinned official Hub release. A BGE checkpoint is rejected. + +The Qwen router requires **Qwen/Qwen3-VL-Embedding-8B, 4096 dimensions and L2 +normalization**, not Qwen3 text embeddings, CLIP features, padded BGE vectors or +concatenated independent text/image features. The question and its image are +encoded together. Context and target use the same encoder and preprocessing; +only training observations populate Context. No held-out outcomes enter encoder +inputs. The frozen router and ORBIT's existing context cost transformation and +shared evaluation remain unchanged; predicted costs are not certified bills. + +### Local encoder + +The example uses `embeddings.qwen.backend="local"`. Download the full +Qwen3-VL-Embedding-8B encoder separately and set `embeddings.qwen.model_dir`. +This is an **8B encoder**, distinct from the much smaller RouteFM router weight. +Local precision defaults to the checkpoint dtype (`qwen.dtype="auto"`); override +with `float32`, `float16` or `bfloat16` when appropriate for your hardware. +The loader uses the [model-card SentenceTransformer interface](https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B), +passing each input as `{"text": question, "image": local_image_path}`. Text-only +and image-only inputs are supported by the encoder too. Loading is local-only +with remote code disabled; there is no silent download or substitute encoder. +Use a compatible multimodal installation (the published SentenceTransformer +model metadata specifies Sentence Transformers 5.4.0; Transformers needs Qwen3-VL +support, e.g. 4.57.x under ORBIT's existing version range); +an old installation that cannot load the official model must be upgraded in a +separate encoder environment or replaced with the API backend below. The +existing main requirements are not globally upgraded for other methods. + +### Explicit embedding service + +Alternatively replace the `qwen` object in the example with: + +```json +{ + "backend": "api", + "base_url": "https://YOUR-SERVICE/v1", + "model": "YOUR-PINNED-QWEN3-VL-EMBEDDING-8B-DEPLOYMENT", + "api_key_env": "ROUTEFM_EMBEDDING_API_KEY", + "timeout": 60 +} +``` + +Set the named environment variable; never store API keys in a committed config. +This **explicitly uploads questions and image bytes** to the configured service +and may incur provider charges. API mode requires a RouteFM-compatible service +accepting `POST BASE_URL/embeddings` with this exact request contract: + +```json +{ + "model": "DEPLOYMENT", + "input": [{"text": "Question", "image": "data:image/png;base64,..."}] +} +``` + +The service must jointly encode each object and return indexed +`{"data":[{"index":0,"embedding":[...]}]}` rows. An optional `qwen.instruction` +is passed as `input[i].instruction` (local mode uses the model's `prompt` +argument); without it the encoder/provider must use its official default. +This image-input format is **not a universal OpenAI or vLLM REST schema**. Use a +compatible provider or a wrapper around the official encoder; a generic text +embedding endpoint is insufficient. Pin the served encoder revision and +preprocessing across all Context/target calls. The client cannot independently +certify what weights a remote provider runs. + +Images must be existing local files; API uploads accept JPEG/PNG/WebP, detect +MIME from image contents (including `.image` files), and reject missing images +and uploads over 32 MiB per image. Remote image URLs are not fetched. HTTPS is +required except for a loopback service, redirects are rejected, and endpoint +credentials in the URL are forbidden. Row identities, dimensions, finite values +and nonzero norms are checked, and output is L2-normalized. Failures never +silently switch to text-only routing. + +See the [official RouteFM custom-data contract](https://github.com/LAMDA-Model-Reuse/RouteFM/blob/main/docs/CUSTOM_DATA.md). + +## ProfileRouter: aligned multimodal extension + +```bash +python main.py --dataset VLRouterBench --method ProfileRouter \ + --config configs/examples/profile_multimodal.json --device cuda:0 +``` + +The default text-only Longformer/profile path remains unchanged. Image mode +selects the matched pretrained CLIP ViT-B/16 text and image encoders. Public +model, family and task descriptions are encoded with its text branch, and the +existing training-free graph propagation produces 512-D model profiles. A query +uses, with each branch separately L2-normalized: + +`q = normalize(alpha * text(question) + (1-alpha) * image(picture))` + +The default `alpha=0.5` is fixed before evaluation. Configure it through +`profile.multimodal_embeddings.text_weight` with `0 < alpha < 1`; do not tune +against test outcomes. Query and profile remain in the same aligned space and +dimension. Cosine scoring and the shared, training-only cost MLP are unchanged. +Description-only calls do not require dummy images. Unpaired encoder names, +unequal dimensions, random trainable projections and invalid weights are +rejected. Custom encoder pairs under `profile.multimodal_embeddings` must +actually share a pretrained aligned space; equal vector sizes alone are not +proof of alignment. CLIP's text tokenizer truncates long descriptions/prompts +to its context window, unlike the original Longformer path. + +This is an **ORBIT multimodal extension**, not an original-paper multimodal +reproduction. It adds no query-reward supervision to profile construction and +does not replace descriptions with held-out benchmark outcomes. Original +[RouteProfile](https://arxiv.org/abs/2605.00180) results and this extension should +be reported separately. + +## Smoke boundaries + +```bash +# Offline: all 28 image-compatible routers, tiny official Qwen architecture. +python scripts/smoke_all_methods.py --benchmarks VLRouterBench --multimodal \ + --methods Avengers AvengersPro CarrotRouter EARAMRouter Eagle EmbedLLM \ + EquiRouter GraphRouter HybridLLM InferenceDynamics MIRT NIRT OmniRouter \ + ProfileRouter RMClassification RMInterval RMSoftmax RouteFM RouteLLM_MF \ + RouteLLM_SWRanking RouterDC SaveRouter TRouter UniRoute knn mlp oracle svm \ + --tiny-routefm --report-dir /tmp/orbit-vl-multimodal-smoke + +# Real bounded source records and released Qwen router weights, NOT real Qwen +# semantic embeddings: input-boundary surrogates remain explicitly recorded. +python scripts/smoke_all_methods.py --real-data --vl-source /path/to/prepared \ + --benchmarks VLRouterBench --multimodal --methods RouteFM ProfileRouter \ + --routefm-checkpoint /path/to/official/routefm_qwen.pt \ + --report-dir /tmp/orbit-vl-qwen-profile-smoke +``` + +The smoke runner uses a deterministic 4096-D text/image-dependent surrogate at +Qwen's encoder boundary, not genuine Qwen embeddings. Profile smoke uses +dimension-matched hash/pixel surrogates, not actual aligned CLIP semantics. +Router/context construction, cost prediction, allocation, metrics and JSON +writing are not mocked. Reports record substitutions. Unit tests separately +exercise real loopback HTTP transport, request image contents, row order and +input validation. A substituted run cannot certify production encoder loading +or routing quality. For real pretrained semantics, use `main.py` with the +examples above and the required encoder resources. + +ModelSAT and RouteLLM_BERT remain outside the multimodal smoke list. CarrotRouter +and EA-RAM are included; EA-RAM's optional response-based ex-post evaluator is +not fitted on VL-RouterBench exports without released response strings. diff --git a/docs/new_benchmarks.md b/docs/new_benchmarks.md index 2d8088d..1290982 100644 --- a/docs/new_benchmarks.md +++ b/docs/new_benchmarks.md @@ -1,5 +1,9 @@ # New benchmark adapters +For **VL-RouterBench**, including its required one-time raw archive preparation, +see the [dedicated adapter guide](vl_routerbench.md). The three adapters below +retain their existing download and evaluation protocols. + All three adapters are registered `BaseDatasetLoader` subclasses. They return the existing `(train_df, test_df, model_list)` tuple with `prompt`, `eval_name`, `model_i_performance` and `model_i_cost` columns. No router, embedding interface, @@ -238,8 +242,8 @@ and not an exhaustive run of all methods on all full datasets. ### All-method runtime validation `tests/test_all_methods_smoke.py` adds a CPU-only matrix covering every registered -method on all three source-schema fixtures, plus all 24 compatible multimodal -methods on MMR image fixtures. It calls real training, prediction, allocation, +method on all four source-schema fixtures (including VL-RouterBench), plus all 24 +compatible multimodal methods on MMR and VL-RouterBench image fixtures. It calls real training, prediction, allocation, nAUC, Peak Score and JSON output code; none of these are mocked. CI discovers this test through the existing unittest entry point. @@ -253,6 +257,7 @@ python scripts/smoke_all_methods.py --tiny-routefm \ # Bounded real benchmark records and an existing released local RouteFM weight. python scripts/smoke_all_methods.py --real-data \ + --benchmarks XRouteBench MMRBenchV2 R2Bench \ --routefm-checkpoint /path/to/routefm_bge.pt \ --report-dir /tmp/orbit-all-methods-real ``` diff --git a/docs/vl_routerbench.md b/docs/vl_routerbench.md new file mode 100644 index 0000000..7088ed9 --- /dev/null +++ b/docs/vl_routerbench.md @@ -0,0 +1,202 @@ +# VL-RouterBench in ORBIT + +ORBIT supports [VL-RouterBench](https://github.com/K1nght/VL-RouterBench) through +the existing `BaseDatasetLoader` contract, retaining the standard router, +training and evaluation interfaces. Dataset adaptation does not change metrics. + +The upstream benchmark describes 14 datasets and 17 VLM candidates. Its public +[Hugging Face release](https://huggingface.co/datasets/KinghtH/VL-RouterBench/tree/6e44fbf50e170f8e72734b49f63f2b52a8866377) +is a roughly 6.87 GB raw archive, **not** a tabular Hugging Face routing dataset. +The HF viewer displays the repository's illustrations, not the routing records. +One-time upstream preparation is therefore required. + +## Prepare the data + +Install ORBIT's normal dependencies. The **upstream preparation** tools also +require `openpyxl`, `tiktoken` and `PyYAML` (see upstream requirements/setup). +They are not new ORBIT runtime dependencies: the loader reads JSONL, Parquet, +CSV and TSV exports and does not import the upstream package. + +Use a new or empty preparation directory: + +If a download was interrupted, a directory containing only the helper's `hub` +cache may be reused for resumption; existing extracted/prepared data is never +overwritten by the helper. + +```bash +# Hash-check the frozen archive and extract with ORBIT's validated TAR helper. +# Reserve space for both the archive and the expanded image/inference assets. +python scripts/download_vlrouterbench.py --output-dir ./data/vl_routerbench + +# Or reuse a previously downloaded official archive: +python scripts/download_vlrouterbench.py \ + --archive /path/to/vlm_router_data.tar.gz \ + --output-dir ./data/vl_routerbench + +# Prepare official named export tables, using the reviewed upstream revision. +git clone https://github.com/K1nght/VL-RouterBench.git /path/to/VL-RouterBench +git -C /path/to/VL-RouterBench switch --detach f5a41ce83d6efe67489b35bb7a07ae72976a9aba +``` + +From the upstream checkout, with an **absolute** path to your preparation root: + +```bash +# Step 2 resolves the model/pricing config relative to the preparation root. +cp -r config /path/to/ORBIT/data/vl_routerbench/config + +python tools/build_benchmark_from_vlmevalkit.py \ + --vlmevalkit_dir /path/to/ORBIT/data/vl_routerbench/vlm_router_data/VLMEvalKit_evaluation \ + --tsv_base_dir /path/to/ORBIT/data/vl_routerbench/vlm_router_data/TSV_images \ + --config_dir config \ + --output_dir /path/to/ORBIT/data/vl_routerbench + +python tools/compute_token_stats.py \ + --dataset_dir /path/to/ORBIT/data/vl_routerbench \ + --vlmevalkit_dir /path/to/ORBIT/data/vl_routerbench/vlm_router_data/VLMEvalKit_inference \ + --output_dir /path/to/ORBIT/data/vl_routerbench/reports/token_statistics +``` + +The upstream scripts are separate software: review their dependency and input +handling before running them. They may consume serialized raw evaluation files; +only use the verified official release or sources you trust. ORBIT's loader +and download helper **never unpickle data or import/execute upstream code**. +Steps 3–6 are not required for the ORBIT adapter; it never reads `Y.npz`, `C.npy` +or pickle index files, so positional matrix alignment and default-cost fallbacks +cannot silently enter the ORBIT evaluation. + +Expected preparation root: + +```text +BENCHMARKS//_samples.jsonl +ORACLE/score/.parquet +SPLITS/{train,dev,test}.jsonl +reports/token_statistics/token_stats_per_sample.csv +reports/token_statistics/output_tokens_per_model.csv +vlm_router_data/TSV_images/.tsv +``` + +Missing exports produce an actionable error; the loader does not silently +download the complete archive, generate new VLM outcomes, or invoke paid APIs. +For an existing prepared directory, set `dataset.source_dir` in the benchmark +config. `dataset.dataset_dir` remains the output/cache/protocol directory. + +## Run existing methods + +```bash +python main.py --dataset VLRouterBench --method GraphRouter +python main.py --dataset VLRouterBench --method SaveRouter --device cpu +python main.py --dataset VLRouterBench --method oracle --device cpu +``` + +The default config is `configs/benchmarks/vlrouterbench.json`, with `text+image` +inputs and CLIP embeddings. Prepare the existing embedding checkpoints as usual. +Change `modality` to `text` and disable the image encoder only for an explicitly +labelled **text-only ablation**, including methods with text-only adapters. +Do not describe such runs as multimodal routing. + +Optional settings under `dataset`: + +- `models`: canonical IDs from `configs/datasets/vlrouterbench.json`. +- `benchmarks`: distinct source dataset IDs; all 14 are the default. +- `max_samples`: deterministic cap on input groups **per official train/test + split**, applied before image materialization. It is for smoke/debugging, + not a full-benchmark result. +- `source_dir`: preparation root; defaults to `dataset_dir`. +- `image_root`: directory containing the TSV image files; defaults to + `/vlm_router_data/TSV_images`. +- `multi_image=contact-sheet`, `image_cell_size=512`: explicit multi-image + transformation for ORBIT's existing single-image embedding input. This is not + a claim to reproduce an upstream multi-image encoder's representation. +- `missing_policy=error` (default) or `drop-query`: reject incomplete candidate + support or explicitly retain common complete support, recording exclusions. +- `text_overlap_policy=error` (default) or `drop-query`: text-only official views + reject identical normalized prompts across train/dev/test by default. + `drop-query` explicitly excludes all queries in those cross-split text groups, + recording counts by split, **for both text and multimodal runs** so the same + population filter can be shared when comparing methods. + Different images do not make the text-only inputs distinguishable. The smoke + runner explicitly selects `drop-query`; normal benchmark runs do not silently + change the population. Alternatively use grouped ORBIT in-domain splits. + +For fair comparisons, use the same candidate pool, population policy and actual +train/test ID sets for all methods. Bounded sampling operates on visible-input +groups, which differ between text-only and image views; check sidecar ID hashes +before comparing their scores, or use the complete shared official population. +- `estimated_output_policy=allow` (default), `error`, or `drop-query`: retain + upstream output-length estimates with flags, require counted output lengths, + or mark estimates missing and use `missing_policy=drop-query`. None of these + options makes the input/image token estimates actual provider usage. + +## Evaluation protocol and limitations + +- All joins use source sample/model IDs. Model order is sorted and shared by + every performance, cost, usage and metadata column. Duplicate identities, + invalid scores, nonfinite/negative/fractional token counts and inconsistent + input token totals fail explicitly. Missing scores/costs are never zero-filled. +- Performance is the released `quality_raw` value in `[0,1]`, without another + integer cast or threshold. **All-zero-performance queries are retained**; + `trainable_sample_ids.txt` is not consulted. ORBIT cannot undo any score + conversion already performed by the upstream exporter. +- Official mode uses upstream `SPLITS/train.jsonl` and `test.jsonl`, excluding + dev from supervision/evaluation. Duplicate assignments, unassigned samples + and identical visible inputs across train/dev/test (declared prompt+image + references for multimodal, normalized prompt alone for text-only) + fail before bounded subsampling. The actual exported split IDs are hashed; + this is not a claim that rerunning the upstream exporter regenerates the + paper's original split (its shuffle also depends on source traversal order). +- Existing grouped `in-domain` and task-based `out-of-domain` ORBIT splits can + be selected explicitly. Their results must be labelled as ORBIT protocols. +- Cost is `(total_input_tokens * input_price + output_tokens * output_price) / + 1e6`, using the frozen upstream reference prices, **not current API prices or + verified provider bills**. Input/image tokenization is an estimate; output + lengths may also be estimated. `model_i_output_is_actual`, token columns, + `model_i_raw_cost`, source-table hashes and estimate counts remain available. + `is_actual=True` only means upstream counted a released response instead of + using a fallback length; it does **not** certify provider usage or per-model + tokenization. Selected materialized image contents are also hashed in the sidecar. + A single training-only maximum scales all cost columns by default. +- No gold answers, score-derived labels, model responses or cost information + become router inputs. The standard exports do not retain response strings; + EA-RAM's optional ex-post evaluator therefore cannot be trained from them. + Its allocation/cost evaluation can still run using its normal no-response path. +- TSV image row references are **zero-based row ordinals** from the exporter, + not numeric values of the TSV `index` field. Short image values reference + another TSV index; these references are resolved separately, with missing, + ambiguous and cyclic references rejected. Base64 image lists preserve order. + Materialized images preserve the original encoded bytes; JPEGs are not + inflated into PNG caches. PIL detects their format by content. +- Author-machine absolute TSV paths are relocated by basename under + `image_root`. Direct image URIs must be valid relative paths under that root; + absolute direct image paths require an explicit export repair. Remote image + URLs and unsupported asset types do not silently degrade to blank images. +- ORBIT reports its existing nAUC, Peak Score and QNC. Upstream Rank Score and + throughput are not replaced by these and are not claimed as reproduced. + +## Local smoke + +```bash +# All 30 methods, real training/prediction/metrics/JSON, source-schema fixtures. +python scripts/smoke_all_methods.py --benchmarks VLRouterBench --tiny-routefm \ + --report-dir /tmp/orbit-vl-text-smoke + +# Image-compatible methods only; choose them through --methods. +python scripts/smoke_all_methods.py --benchmarks VLRouterBench --multimodal \ + --methods GraphRouter SaveRouter knn oracle \ + --report-dir /tmp/orbit-vl-image-smoke + +# Prepared actual records, bounded to 32 train/test input groups and two models. +python scripts/smoke_all_methods.py --real-data --vl-source /path/to/prepared \ + --benchmarks VLRouterBench --methods GraphRouter SaveRouter knn oracle \ + --multimodal --report-dir /tmp/orbit-vl-real-smoke +``` + +The smoke runner uses deterministic input-only hash text features and actual +image-derived features. Large transformer checkpoints are replaced by tiny local +models and NIRT's paid annotation boundary is stubbed. These are runtime checks, +not paper-quality or full-benchmark reproduction claims. CI additionally tests +every registered method on all four source-schema fixtures, and 28 compatible +methods on each MMR V2/VL-RouterBench multimodal fixture. +RouteFM multimodal smoke uses Qwen router weights with an explicitly labeled +joint encoder surrogate; ProfileRouter uses aligned-fusion surrogates. For +production encoder configuration and limitations, see +[multimodal adapters](multimodal_adapters.md). diff --git a/main.py b/main.py index 404f168..0889eca 100644 --- a/main.py +++ b/main.py @@ -51,6 +51,7 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--dataset", type=str, required=True, help="Benchmark dataset name (case-insensitive).") parser.add_argument("--method", type=str, required=True, help="Router method config name (case-insensitive, without .json).") + parser.add_argument("--config", type=Path, help="Optional top-level JSON overrides (e.g. configs/examples/routefm_qwen.json).") parser.add_argument( "--device", type=str, @@ -60,6 +61,10 @@ def main() -> None: args_cli = parser.parse_args() config = load_config(args_cli.dataset, args_cli.method, device=args_cli.device) + if args_cli.config is not None: + config = merge_dict_shallow(config, load_json(args_cli.config)) + if args_cli.device is not None: + config["device"] = args_cli.device train(config) diff --git a/methods/ProfileRouter.py b/methods/ProfileRouter.py index b801ad2..f404970 100644 --- a/methods/ProfileRouter.py +++ b/methods/ProfileRouter.py @@ -13,6 +13,7 @@ import json import re from pathlib import Path +from copy import deepcopy import numpy as np import torch @@ -30,6 +31,17 @@ class ProfileRouter(BaseRouter): """Embedding-GNN RouteProfile combined with the paper's SimRouter.""" def __init__(self, args): + args = deepcopy(args) + if "image" in args["modality"].split("+"): + # Explicit multimodal extension, not the paper's Longformer path. + multimodal = dict(args.get("profile", {}).get("multimodal_embeddings", {})) + args["embeddings"] = { + "text_model": "ViT-B/16", "image_model": "ViT-B/16", + "out_dim": 512, "batch_size": 8, + **multimodal, "fusion": "aligned_mean", "training": False, "normalize": True, + } + if args["embeddings"]["text_model"] != args["embeddings"]["image_model"]: + raise ValueError("ProfileRouter requires a matched aligned text/image encoder pair.") super().__init__(args) profile_cfg = dict(args.get("profile", {})) self.aggregation_hops = int(profile_cfg.get("aggregation_hops", 4)) @@ -256,6 +268,8 @@ def predict(self, test_embedding): if self.model_profiles is None: raise RuntimeError("ProfileRouter must be trained before prediction.") query_features = _as_numpy(test_embedding) + if query_features.ndim != 2 or query_features.shape[1] != self.model_profiles.shape[1]: + raise ValueError("ProfileRouter query/profile dimensions differ; use aligned multimodal fusion.") query_norms = np.linalg.norm(query_features, axis=1, keepdims=True) normalized_queries = query_features / np.maximum(query_norms, 1e-12) similarities = normalized_queries @ self.model_profiles.T diff --git a/methods/RouteFM.py b/methods/RouteFM.py index b69f239..f3465c3 100644 --- a/methods/RouteFM.py +++ b/methods/RouteFM.py @@ -21,17 +21,22 @@ class RouteFMRouter(BaseRouter): def __init__(self, args): route_config = dict(args.get("routefm", {})) encoder = str(route_config.get("encoder", "bge")).lower() - if encoder != "bge": - raise ValueError( - "ORBIT's built-in RouteFM adapter currently supports the official " - "BGE text checkpoint. The Qwen checkpoint requires joint Qwen3-VL " - "embeddings from a compatible service." - ) + if encoder not in {"bge", "qwen"}: + raise ValueError("RouteFM encoder must be bge or qwen.") # The released BGE checkpoint is text-only. In a multimodal benchmark, # this is an explicit text-only evaluation rather than concatenating an # incompatible image representation. args = dict(args) - args["modality"] = "text" + if encoder == "bge": + args["modality"] = "text" + else: + embeddings = dict(args.get("embeddings", {})) + if embeddings.get("joint_model") != "Qwen/Qwen3-VL-Embedding-8B": + raise ValueError("RouteFM-Qwen requires the Qwen3-VL-Embedding-8B joint encoder.") + if embeddings.get("out_dim") != 4096 or embeddings.get("training", False): + raise ValueError("RouteFM-Qwen requires frozen 4096-D embeddings without projection.") + embeddings["normalize"] = True + args["embeddings"] = embeddings super().__init__(args) self.route_config = route_config self.encoder = encoder diff --git a/requirements-ci.txt b/requirements-ci.txt index c0804ab..3281589 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -3,6 +3,7 @@ # are intentionally excluded. torch-geometric>=2.6,<2.8 transformers>=4.45,<4.58 +sentence-transformers>=5.4,<6 datasets>=2.20,<4.5 huggingface-hub>=0.24,<0.36 git+https://github.com/openai/CLIP.git diff --git a/scripts/download_vlrouterbench.py b/scripts/download_vlrouterbench.py new file mode 100644 index 0000000..918481c --- /dev/null +++ b/scripts/download_vlrouterbench.py @@ -0,0 +1,57 @@ +"""Download and safely unpack the frozen raw release; never execute pickles. + +Upstream Steps 1/2 are a separate, explicit preparation step. This helper does +not install/run upstream code or evaluate VLMs. See docs/vl_routerbench.md. +""" + +import argparse +import hashlib +from pathlib import Path +import shutil +import sys +import tarfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +REVISION = "6e44fbf50e170f8e72734b49f63f2b52a8866377" +ARCHIVE_SHA256 = "8b52f86e8f6679c300dfd12212f7c6a520840d9906afc1587e887039587dfddc" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, default=ROOT / "data/vl_routerbench", + help="New or empty preparation directory.") + parser.add_argument("--archive", type=Path, help="Previously downloaded official archive (hash checked).") + args = parser.parse_args() + directory = args.output_dir.resolve() + if directory.exists() and {p.name for p in directory.iterdir()} - {"hub"}: + parser.error("--output-dir must be new/empty or contain only a resumable hub cache; existing data will not be overwritten.") + directory.mkdir(parents=True, exist_ok=True) + archive_path = args.archive + if archive_path is None: + from huggingface_hub import hf_hub_download + archive_path = Path(hf_hub_download( + repo_id="KinghtH/VL-RouterBench", repo_type="dataset", revision=REVISION, + filename="vlm_router_data.tar.gz", cache_dir=str(directory / "hub"), + )) + digest = hashlib.sha256() + with archive_path.open("rb") as handle: + for chunk in iter(lambda: handle.read(4 * 1024 * 1024), b""): + digest.update(chunk) + if digest.hexdigest() != ARCHIVE_SHA256: + raise ValueError("Archive hash does not match the frozen official release; nothing extracted.") + from utils.data import safe_extract_tar + with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + expanded = sum(member.size for member in members if member.isfile()) + if shutil.disk_usage(directory).free < expanded + 256 * 1024 * 1024: + raise OSError(f"Insufficient disk space for {expanded} bytes of extracted data.") + safe_extract_tar(archive, str(directory)) + print(f"Verified and safely extracted the raw release to {directory}.") + print("Next: run upstream Steps 1 and 2 as documented in docs/vl_routerbench.md.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_all_methods.py b/scripts/smoke_all_methods.py index b7288d9..f103c37 100644 --- a/scripts/smoke_all_methods.py +++ b/scripts/smoke_all_methods.py @@ -34,13 +34,14 @@ from smoke_new_benchmarks import _stream_r2_prefix # registers smoke-hash from main import load_config from train import ROUTER_REGISTRY, create_router -from utils.embedding import IMAGE_EMBEDDERS +from utils.embedding import IMAGE_EMBEDDERS, TEXT_EMBEDDERS from test_new_benchmarks import ( _args, _write_sources, _x_sources, _m_sources, _r_sources, X_MODELS, M_MODELS, R_MODELS, ) +from test_vl_routerbench import VL_MODELS, vl_args, write_vl_fixture -TEXT_ONLY = {"ModelSAT", "RouteLLM_BERT", "RouteFM", "ProfileRouter", "CarrotRouter", "EARAMRouter"} +TEXT_ONLY = {"ModelSAT", "RouteLLM_BERT"} PAIR_METHODS = {"HybridLLM", "RouteLLM_SWRanking", "RouteLLM_MF", "RouteLLM_BERT"} DIRECT_METHODS = {"RMClassification", "RMSoftmax", "RMInterval"} @@ -58,6 +59,25 @@ def embed(paths): return embed, 8 +@TEXT_EMBEDDERS.register("smoke-aligned") +def _aligned_text(args): + return TEXT_EMBEDDERS.create("smoke-hash", args) + + +@IMAGE_EMBEDDERS.register("smoke-aligned") +def _aligned_image(args): + # Shape/flow surrogate only: not a pretrained aligned semantic space. + return _image_embedder(args) + + +def _joint_surrogate(self, inputs): + text, _ = TEXT_EMBEDDERS.create("smoke-hash", {}) + image, _ = _image_embedder({}) + values = torch.cat((text([item.get("text", "") for item in inputs]), + image([item["image"] for item in inputs])), dim=1) + return values.repeat(1, 256).numpy() + + def _tiny_transformers(directory): from tokenizers import Tokenizer, models, pre_tokenizers from transformers import ( @@ -79,11 +99,11 @@ def _tiny_transformers(directory): max_position_embeddings=512)).save_pretrained(directory / "qwen") -def _tiny_routefm(path): +def _tiny_routefm(path, encoder="bge"): """Real official architecture, random tiny weights: portable offline CI.""" from dataclasses import asdict from routefm.models import RouteFM, RouteFMConfig - config = RouteFMConfig(query_dim=8, hidden_dim=8, projection_dim=8, + config = RouteFMConfig(query_dim=4096 if encoder == "qwen" else 8, hidden_dim=8, projection_dim=8, ffn_dim=16, heads=2, profile_layers=1, readout_layers=1, pool_layers=1, capability_tokens=2, quality_bins=5, observation_features=2, observation_schema="score_cost", @@ -92,17 +112,19 @@ def _tiny_routefm(path): torch.manual_seed(42) model = RouteFM(config).eval() path.parent.mkdir(parents=True, exist_ok=True) - torch.save({"format": "routefm_unified_scratch", "encoder": "bge", + torch.save({"format": "routefm_unified_scratch", "encoder": encoder, "model_config": asdict(config), "model": model.state_dict()}, path) -def _prepare(directory, real): +def _prepare(directory, real, vl_source=None, benchmarks=None): configs = {} for name, source, options in ( ("XRouteBench", _x_sources(), {"models": X_MODELS}), ("MMRBenchV2", _m_sources(), {"models": M_MODELS, "benchmarks": ["BLINK"]}), ("R2Bench", _r_sources(), {"models": R_MODELS, "budgets": [10, 100]}), ): + if benchmarks is not None and name not in benchmarks: + continue data_root = directory / "data" / name if real: args = load_config(name, "knn", device="cpu") @@ -124,6 +146,20 @@ def _prepare(directory, real): # EA-RAM's optional evaluator requires genuine released response fields. args["dataset"]["include_responses"] = True configs[name] = args + if (benchmarks is None or "VLRouterBench" in benchmarks) and (not real or vl_source is not None): + data_root = directory / "data" / "VLRouterBench" + if real: + args = load_config("VLRouterBench", "knn", device="cpu") + args["description_path"] = str(ROOT / args["description_path"]) + args["dataset"].update(dataset_dir=str(data_root), source_dir=str(vl_source), + max_samples=32, models=VL_MODELS, benchmarks=["RealWorldQA"]) + else: + write_vl_fixture(data_root) + args = vl_args(data_root) + # Keep upstream assignments but explicitly exclude indistinguishable + # text-only inputs that cross them; no test outcomes guide this filter. + args["dataset"]["text_overlap_policy"] = "drop-query" + configs["VLRouterBench"] = args return configs @@ -186,12 +222,16 @@ def _run(base, method, directory, checkpoints, routefm_checkpoint, multimodal, p if method == "ModelSAT": args["model"] = dict(args["model"], base_model=str(checkpoints / "qwen"), lora_r=2, lora_alpha=4) args["training"].update(batch_size=4, accumulation_steps=1, max_length=64) + if method == "ProfileRouter" and multimodal: + args["profile"]["multimodal_embeddings"] = dict( + args["embeddings"], text_model="smoke-aligned", image_model="smoke-aligned", out_dim=8) if method == "RouteFM": if not routefm_checkpoint: raise ValueError("RouteFM needs --routefm-checkpoint; it is not silently skipped.") - args["routefm"] = dict(args["routefm"], checkpoint=str(routefm_checkpoint)) + encoder = "qwen" if multimodal else "bge" + args["routefm"] = dict(args["routefm"], checkpoint=str(routefm_checkpoint), encoder=encoder) from routefm.predict import load_router - dimension = int(load_router(routefm_checkpoint, "cpu", "bge").config.query_dim) + dimension = int(load_router(routefm_checkpoint, "cpu", encoder).config.query_dim) # Repeat the input-only hash features to the checkpoint's real dimension. from utils.embedding import TEXT_EMBEDDERS key = f"smoke-routefm-{dimension}" @@ -204,6 +244,9 @@ def widened(texts): return values.repeat(1, (dimension + 7) // 8)[:, :dimension] return widened, dimension args["embeddings"].update(text_model=key, out_dim=dimension) + if multimodal: + args["embeddings"] = {"joint_model": "Qwen/Qwen3-VL-Embedding-8B", "out_dim": 4096, + "normalize": True, "training": False, "batch_size": 16} name = args["dataset"]["name"] log_path = directory / "logs" / f"{name}-{method}-{'multimodal' if multimodal else 'text'}.log" @@ -212,12 +255,18 @@ def widened(texts): logging.getLogger().addHandler(handler) started = time.monotonic() substitutions = ["input-only hash embeddings", "one epoch"] + if method == "ProfileRouter" and multimodal: + substitutions.append("aligned-fusion shape/flow surrogate; not pretrained CLIP embeddings") if pair is not None and method.startswith("RouteLLM_"): substitutions.append(f"explicit candidate pair strong={pair[0]}, weak={pair[1]}") try: with ExitStack() as stack: progress = stack.enter_context((log_path.with_suffix(".progress.log")).open("w", encoding="utf-8")) stack.enter_context(redirect_stderr(progress)) + if method == "RouteFM" and multimodal: + stack.enter_context(patch("utils.qwen_vl_embedding.QwenVLEmbedder.__init__", return_value=None)) + stack.enter_context(patch("utils.qwen_vl_embedding.QwenVLEmbedder._encode_batch", _joint_surrogate)) + substitutions.append("4096-D input-only joint surrogate; not real Qwen3-VL embeddings") if method == "NIRT": annotation = stack.enter_context(patch.object(importlib.import_module("methods.NIRT"), "get_LLM_response", return_value="Reasoning, Understanding")) substitutions.append("offline auxiliary-LLM annotation boundary") @@ -287,8 +336,9 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--real-data", action="store_true", help="Opt into bounded public source reads; default is offline fixtures.") parser.add_argument("--methods", nargs="+", choices=sorted(ROUTER_REGISTRY), default=sorted(ROUTER_REGISTRY)) - parser.add_argument("--benchmarks", nargs="+", choices=["XRouteBench", "MMRBenchV2", "R2Bench"], default=["XRouteBench", "MMRBenchV2", "R2Bench"]) - parser.add_argument("--multimodal", action="store_true", help="MMR fixture images with compatible methods only, no raw-media downloads.") + parser.add_argument("--benchmarks", nargs="+", choices=["XRouteBench", "MMRBenchV2", "R2Bench", "VLRouterBench"], default=["XRouteBench", "MMRBenchV2", "R2Bench", "VLRouterBench"]) + parser.add_argument("--vl-source", type=Path, help="Existing upstream VL-RouterBench Steps 1/2 exports for --real-data.") + parser.add_argument("--multimodal", action="store_true", help="MMR/VL images with compatible methods; RouteFM uses Qwen weights and a joint encoder surrogate.") parser.add_argument("--routefm-checkpoint", type=Path) parser.add_argument("--tiny-routefm", action="store_true", help="Generate an untrained tiny official-architecture checkpoint for offline CI, not released-weight validation.") parser.add_argument("--pair-models", type=int, nargs=2, metavar=("STRONG", "WEAK"), help="Optional explicit RouteLLM candidate indices; recorded in the report.") @@ -297,8 +347,11 @@ def main(): directory = cli.report_dir.resolve() if directory.exists() and any(directory.iterdir()): parser.error("--report-dir must be new or empty; existing reports are never overwritten.") - if cli.multimodal and (cli.real_data or cli.benchmarks != ["MMRBenchV2"] or set(cli.methods) & TEXT_ONLY): - parser.error("--multimodal requires offline MMRBenchV2 only and methods outside TEXT_ONLY.") + if cli.real_data and "VLRouterBench" in cli.benchmarks and cli.vl_source is None: + parser.error("Real VLRouterBench smoke requires --vl-source pointing to prepared upstream exports.") + if cli.multimodal and (not set(cli.benchmarks) <= {"MMRBenchV2", "VLRouterBench"} + or (cli.real_data and "MMRBenchV2" in cli.benchmarks) or set(cli.methods) & TEXT_ONLY): + parser.error("--multimodal supports MMRBenchV2 fixtures or VLRouterBench and methods outside TEXT_ONLY.") directory.mkdir(parents=True, exist_ok=True) checkpoint = cli.routefm_checkpoint.resolve() if cli.routefm_checkpoint else None if cli.tiny_routefm and checkpoint is not None: @@ -307,13 +360,14 @@ def main(): torch.set_num_interop_threads(1) logging.getLogger().setLevel(logging.INFO) os.chdir(ROOT) - configurations = _prepare(directory, cli.real_data) + configurations = _prepare(directory, cli.real_data, cli.vl_source.resolve() if cli.vl_source else None, + benchmarks=cli.benchmarks) transformer_dir = directory / "tiny_checkpoints" if {"ModelSAT", "RouteLLM_BERT"} & set(cli.methods): _tiny_transformers(transformer_dir) if cli.tiny_routefm: checkpoint = transformer_dir / "tiny_routefm.pt" - _tiny_routefm(checkpoint) + _tiny_routefm(checkpoint, "qwen" if cli.multimodal else "bge") results = [] os.chdir(directory) for name in cli.benchmarks: diff --git a/tests/test_all_methods_smoke.py b/tests/test_all_methods_smoke.py index 7635072..5c03e98 100644 --- a/tests/test_all_methods_smoke.py +++ b/tests/test_all_methods_smoke.py @@ -38,6 +38,8 @@ def test_every_registered_method_trains_predicts_and_writes_evaluation(self): _tiny_transformers(checkpoints) routefm = checkpoints / "tiny_routefm.pt" _tiny_routefm(routefm) + routefm_qwen = checkpoints / "tiny_routefm_qwen.pt" + _tiny_routefm(routefm_qwen, "qwen") os.chdir(directory) for name, base in configs.items(): for method in sorted(ROUTER_REGISTRY): @@ -46,10 +48,12 @@ def test_every_registered_method_trains_predicts_and_writes_evaluation(self): # no manually chosen endpoints conceal regressions. result = _run(base, method, directory, checkpoints, routefm, False) self.assertEqual(result["status"], "PASS") - for method in sorted(set(ROUTER_REGISTRY) - TEXT_ONLY): - with self.subTest(benchmark="MMRBenchV2", method=method, modality="text+image"): - result = _run(configs["MMRBenchV2"], method, directory, checkpoints, routefm, True) - self.assertEqual(result["status"], "PASS") + for benchmark in ("MMRBenchV2", "VLRouterBench"): + for method in sorted(set(ROUTER_REGISTRY) - TEXT_ONLY): + with self.subTest(benchmark=benchmark, method=method, modality="text+image"): + result = _run(configs[benchmark], method, directory, checkpoints, + routefm_qwen if method == "RouteFM" else routefm, True) + self.assertEqual(result["status"], "PASS") finally: os.chdir(previous_directory) logger.setLevel(previous_level) diff --git a/tests/test_multimodal_adapters.py b/tests/test_multimodal_adapters.py new file mode 100644 index 0000000..64c8f22 --- /dev/null +++ b/tests/test_multimodal_adapters.py @@ -0,0 +1,221 @@ +"""Multimodal encoder contracts; no pretrained encoder downloads or paid API calls.""" + +import base64 +from contextlib import contextmanager +import io +import json +import os +from pathlib import Path +import tempfile +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import Mock, patch + +import numpy as np +from PIL import Image +import torch + +from main import main +from methods.ProfileRouter import ProfileRouter +from methods.RouteFM import RouteFMRouter +from utils.embedding import Embedder +from utils.qwen_vl_embedding import QwenVLEmbedder, _NoRedirect + + +class AlignedFusionTest(unittest.TestCase): + def _encoder(self, **options): + args = {"embeddings": {"text_model": "pair", "image_model": "pair", "out_dim": 2, + "batch_size": 2, "normalize": True, "training": False, **options}} + text = lambda rows: torch.tensor([[3., 0.] for _ in rows]) + image = lambda rows: torch.tensor([[0., float(value)] for value in rows]) + with patch("utils.embedding.CreateTextEmbedder", return_value=(text, 2)), patch( + "utils.embedding.CreateImageEmbedder", return_value=(image, 2) + ): + return Embedder(args) + + def test_default_concat_is_unchanged(self): + encoder = self._encoder() + result = encoder.run_embed(["same"], [4]) + np.testing.assert_allclose(result, [[.6, 0., 0., .8]]) + + def test_aligned_weighted_unit_vectors_and_image_sensitivity(self): + encoder = self._encoder(fusion="aligned_mean", text_weight=.75) + result = encoder.run_embed(["same", "same"], [4, -4]) + expected = torch.tensor([[.75, .25], [.75, -.25]]) + expected /= expected.norm(dim=1, keepdim=True) + torch.testing.assert_close(result, expected) + self.assertFalse(torch.equal(result[0], result[1])) + self.assertEqual(encoder.run_embed(["model description"]).shape, (1, 2)) + + def test_rejects_random_projections_invalid_weights_and_mismatched_dimensions(self): + for options in ({"training": True}, {"text_weight": 0.0}, {"text_weight": 1.0}, + {"text_weight": float("nan")}): + with self.subTest(options=options), self.assertRaises(ValueError): + self._encoder(fusion="aligned_mean", **options) + with patch("utils.embedding.CreateTextEmbedder", return_value=(Mock(), 2)), patch( + "utils.embedding.CreateImageEmbedder", return_value=(Mock(), 3) + ), self.assertRaisesRegex(ValueError, "equal dimension"): + Embedder({"embeddings": {"text_model": "a", "image_model": "b", "fusion": "aligned_mean"}}) + + def test_profile_multimodal_chooses_clip_without_mutating_caller(self): + args = {"modality": "text+image", "embeddings": {"text_model": "longformer"}, + "description_path": "metadata.json"} + def initialize(router, received): + router.received = received + with patch("methods.base.BaseRouter.__init__", initialize): + router = ProfileRouter(args) + em = router.received["embeddings"] + self.assertEqual((em["text_model"], em["image_model"], em["out_dim"]), ("ViT-B/16", "ViT-B/16", 512)) + self.assertEqual(em["fusion"], "aligned_mean") + self.assertEqual(args["embeddings"], {"text_model": "longformer"}) + + def test_aligned_fusion_rejects_empty_image_signal(self): + encoder = self._encoder(fusion="aligned_mean") + with self.assertRaisesRegex(ValueError, "each modality"): + encoder.run_embed(["question"], [0]) + + def test_profile_rejects_unmatched_encoder_names(self): + with self.assertRaisesRegex(ValueError, "matched aligned"): + ProfileRouter({"modality": "text+image", "profile": {"multimodal_embeddings": { + "text_model": "longformer", "image_model": "ViT-B/16"}}}) + + def test_profile_reports_dimension_contract(self): + router = ProfileRouter.__new__(ProfileRouter) + router.model_profiles = np.ones((2, 8), dtype=np.float32) + with self.assertRaisesRegex(ValueError, "query/profile dimensions"): + router.predict(np.ones((3, 16), dtype=np.float32)) + + +@contextmanager +def _embedding_service(): + received = [] + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + payload = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + received.append((self.path, self.headers.get("Authorization"), payload)) + rows = [] + for index, item in enumerate(payload["input"]): + raw = base64.b64decode(item["image"].split(",", 1)[1]) + with Image.open(io.BytesIO(raw)) as image: + rgb = np.asarray(image.convert("RGB"), dtype=np.float32).mean(axis=(0, 1)) + vector = np.zeros(4096, dtype=np.float32) + vector[:3] = rgb + 1 + rows.append({"index": index, "embedding": vector.tolist()}) + body = json.dumps({"data": list(reversed(rows))}).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + worker = threading.Thread(target=lambda: server.serve_forever(poll_interval=.01), daemon=True) + worker.start() + try: + yield f"http://127.0.0.1:{server.server_port}/v1", received + finally: + server.shutdown() + server.server_close() + worker.join() + + +class QwenJointTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.paths = [self.root / "red.image", self.root / "green.image"] + for path, color in zip(self.paths, ("red", "green")): + Image.new("RGB", (4, 4), color=color).save(path, format="PNG") + + def tearDown(self): + self.temporary.cleanup() + + def _config(self, **qwen): + return {"joint_model": "Qwen/Qwen3-VL-Embedding-8B", "out_dim": 4096, + "normalize": True, "batch_size": 2, "qwen": qwen} + + def test_real_http_joint_payload_order_normalization_and_image_sensitivity(self): + with _embedding_service() as (base_url, received), patch.dict(os.environ, {"ORBIT_TEST_KEY": "fixture-only"}): + encoder = Embedder({"embeddings": self._config(backend="api", base_url=base_url, api_key_env="ORBIT_TEST_KEY")}) + result = encoder.run_embed(["same question"] * 2, self.paths) + self.assertEqual(result.shape, (2, 4096)) + torch.testing.assert_close(result.norm(dim=1), torch.ones(2)) + self.assertGreater(result[0, 0], result[0, 1]) + self.assertGreater(result[1, 1], result[1, 0]) + self.assertEqual(received[0][0], "/v1/embeddings") + self.assertEqual(received[0][1], "Bearer fixture-only") + self.assertEqual(received[0][2]["input"][0]["text"], "same question") + self.assertTrue(received[0][2]["input"][0]["image"].startswith("data:image/png;base64,")) + + def test_local_uses_official_text_image_input_interface(self): + local = self.root / "local-model" + local.mkdir() + model = Mock() + model.encode.return_value = np.ones((2, 4096), dtype=np.float32) + with patch("sentence_transformers.SentenceTransformer", return_value=model) as factory: + encoder = QwenVLEmbedder(self._config(backend="local", model_dir=str(local))) + result = encoder(["question"] * 2, self.paths) + self.assertEqual(result.shape, (2, 4096)) + self.assertTrue(factory.call_args.kwargs["local_files_only"]) + self.assertFalse(factory.call_args.kwargs["trust_remote_code"]) + self.assertEqual(factory.call_args.kwargs["model_kwargs"], {"torch_dtype": "auto"}) + sent = model.encode.call_args.args[0] + self.assertEqual(sent[0], {"text": "question", "image": str(self.paths[0])}) + + def test_missing_local_weights_does_not_download_or_fallback(self): + with self.assertRaisesRegex(FileNotFoundError, "No substitute"): + QwenVLEmbedder(self._config(backend="local", model_dir=str(self.root / "missing"))) + + def test_api_rejects_invalid_indices_shapes_values_and_missing_images(self): + with patch.dict(os.environ, {"ORBIT_TEST_KEY": "fixture-only"}): + encoder = QwenVLEmbedder(self._config(backend="api", base_url="http://localhost/v1", api_key_env="ORBIT_TEST_KEY")) + responses = [ + {"data": [{"index": 0, "embedding": [1.] * 4096}] * 2}, + {"data": [{"index": True, "embedding": [1.] * 4096}]}, + {"data": [{"index": 0, "embedding": [1.] * 8}]}, + {"data": [{"index": 0, "embedding": [0.] * 4096}]}, + {"data": [{"index": 0, "embedding": [float("nan")] * 4096}]}, + ] + for payload in responses: + with self.subTest(payload_length=len(payload["data"])), patch.dict(os.environ, {"ORBIT_TEST_KEY": "fixture-only"}), patch.object( + encoder.opener, "open", return_value=io.BytesIO(json.dumps(payload).encode()) + ), self.assertRaises(ValueError): + encoder(["question"], [self.paths[0]]) + with self.assertRaisesRegex(ValueError, "local files"): + encoder(["question"], ["https://example.org/image.png"]) + with self.assertRaisesRegex(ValueError, "batch sizes"): + encoder(["question"], self.paths) + + def test_joint_encoder_rejects_separate_encoders_projections_and_dimension_changes(self): + for extra in ({"text_model": "ViT-B/16"}, {"training": True}, {"out_dim": 512}): + with self.subTest(extra=extra), patch("utils.embedding.JOINT_EMBEDDERS.create", return_value=(Mock(), 4096)), self.assertRaises(ValueError): + Embedder({"embeddings": {**self._config(), **extra}}) + + def test_routefm_qwen_retains_image_modality_and_requires_correct_encoder(self): + args = {"modality": "text+image", "routefm": {"encoder": "qwen"}, "embeddings": self._config()} + with patch("methods.base.BaseRouter.__init__", lambda router, config: setattr(router, "args", config)): + router = RouteFMRouter(args) + self.assertEqual(router.args["modality"], "text+image") + for extra in ({"joint_model": "other"}, {"out_dim": 512}, {"training": True}): + with self.subTest(extra=extra), self.assertRaises(ValueError): + RouteFMRouter({**args, "embeddings": {**args["embeddings"], **extra}}) + + def test_api_security_and_no_redirect(self): + for base_url in ("", "http://remote.example/v1", "https://user:pass@example.org/v1", "https://example.org/v1?key=secret"): + with self.subTest(base_url=base_url), self.assertRaises(ValueError): + QwenVLEmbedder(self._config(backend="api", base_url=base_url)) + self.assertIsNone(_NoRedirect().redirect_request(None, None, 302, "", {}, "https://elsewhere.invalid")) + + def test_cli_override_selects_qwen_but_device_flag_still_wins(self): + with patch("sys.argv", ["main.py", "--dataset", "VLRouterBench", "--method", "RouteFM", "--config", + "configs/examples/routefm_qwen.json", "--device", "cpu"]), patch("main.train") as train: + main() + config = train.call_args.args[0] + self.assertEqual(config["routefm"]["encoder"], "qwen") + self.assertEqual(config["modality"], "text+image") + self.assertEqual(config["device"], "cpu") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_router_implementations.py b/tests/test_router_implementations.py index 7287961..910422d 100644 --- a/tests/test_router_implementations.py +++ b/tests/test_router_implementations.py @@ -164,11 +164,13 @@ def test_pair_configuration_non_tied_automatic_extremes_are_unchanged(self): self.assertEqual((resolved.strong_model_idx, resolved.weak_model_idx), (1, 2)) def test_unavailable_cuda_device_has_safe_fallback(self): - resolved = BaseRouter._resolve_device("cuda:99") - if torch.cuda.is_available(): - self.assertEqual(resolved, torch.device("cuda:0")) - else: - self.assertEqual(resolved, torch.device("cpu")) + # An invalid visible index falls back to CPU, even on a GPU host. + # Mock availability so the test does not depend on the runner's GPUs. + for available in (False, True): + with self.subTest(available=available), patch( + "torch.cuda.is_available", return_value=available + ), patch("torch.cuda.device_count", return_value=1): + self.assertEqual(BaseRouter._resolve_device("cuda:99"), torch.device("cpu")) def test_native_cost_predictions_are_clipped_per_model(self): router = _ConcreteBase.__new__(_ConcreteBase) diff --git a/tests/test_vl_routerbench.py b/tests/test_vl_routerbench.py new file mode 100644 index 0000000..215f537 --- /dev/null +++ b/tests/test_vl_routerbench.py @@ -0,0 +1,317 @@ +"""Prepared upstream export fixtures: no networks, pickles or position joins.""" + +import base64 +from contextlib import redirect_stderr +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd +from PIL import Image + +from main import load_config +from utils.data import BaseDatasetLoader, VLRouterBenchLoader, download_dataset, get_loader + + +ROOT = Path(__file__).resolve().parents[1] +VL_MODELS = ["Janus-Pro-1B", "Janus-Pro-7B"] + + +def write_vl_fixture(root): + root = Path(root) + for directory in ("BENCHMARKS/vqa_oe", "ORACLE/score", "SPLITS", "reports/token_statistics", "vlm_router_data/TSV_images"): + (root / directory).mkdir(parents=True, exist_ok=True) + def encoded(color): + buffer = io.BytesIO() + Image.new("RGB", (20, 16), color).save(buffer, format="PNG") + return base64.b64encode(buffer.getvalue()).decode() + red, blue = encoded("red"), encoded("blue") + pd.DataFrame({"index": [10, 20, 30], "image": [red, "10", repr([red, blue])]}).to_csv( + root / "vlm_router_data/TSV_images/RealWorldQA.tsv", sep="\t", index=False) + samples, scores, inputs, outputs = [], [], [], [] + for i in range(14): + sid = f"RealWorldQA/{i}" + samples.append({"sample_id": sid, "dataset": "RealWorldQA", "prompt": f"vl visual question {i}", + "assets": [{"type": "image_tsv", "tsv_file": "/author/LMUData/RealWorldQA.tsv", "index": i % 3}], + "answer": "GOLD MUST NOT BECOME INPUT", "task_type": "vqa_oe"}) + inputs.append({"sample_id": sid, "text_tokens": 20+i, "image_tokens": 256, "total_input_tokens": 276+i}) + for j, model in enumerate(VL_MODELS): + scores.append({"sample_id": sid, "model_id": model, "quality_raw": 0 if i == 0 else (i+j)%2, "quality": 1}) + outputs.append({"sample_id": sid, "model": model, "output_tokens": 5+3*j, + "is_actual": not (i == 4 and j == 1)}) + (root / "BENCHMARKS/vqa_oe/realworldqa_samples.jsonl").write_text( + "".join(json.dumps(row)+"\n" for row in samples), encoding="utf-8") + pd.DataFrame(scores).sample(frac=1, random_state=5).to_parquet(root / "ORACLE/score/realworldqa.parquet", index=False) + pd.DataFrame(inputs).sample(frac=1, random_state=7).to_csv(root / "reports/token_statistics/token_stats_per_sample.csv", index=False) + pd.DataFrame(outputs).sample(frac=1, random_state=9).to_csv(root / "reports/token_statistics/output_tokens_per_model.csv", index=False) + for name, indices in (("train", range(8)), ("dev", range(8, 10)), ("test", range(10, 14))): + (root / f"SPLITS/{name}.jsonl").write_text( + "".join(json.dumps({"sample_id": f"RealWorldQA/{i}"})+"\n" for i in indices), encoding="utf-8") + + +def vl_args(root, modality="text", **options): + args = load_config("VLRouterBench", "knn", device="cpu") + args["modality"] = modality + args["description_path"] = str(ROOT / args["description_path"]) + args["dataset"].update(dataset_dir=str(root), models=VL_MODELS, + benchmarks=["RealWorldQA"], **options) + return args + + +class VLRouterBenchTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="orbit-vl-fixture-") + self.root = Path(self.temporary.name) + write_vl_fixture(self.root) + + def tearDown(self): + self.temporary.cleanup() + + def _csv(self, name, edit): + path = self.root / f"reports/token_statistics/{name}.csv" + frame = pd.read_csv(path) + edit(frame).to_csv(path, index=False) + + def test_registry_and_normal_loader_contract(self): + self.assertIs(get_loader("VLRouterBench"), VLRouterBenchLoader) + self.assertTrue(issubclass(VLRouterBenchLoader, BaseDatasetLoader)) + train, test, models = download_dataset(vl_args(self.root)) + self.assertEqual(models, VL_MODELS) + self.assertEqual((len(train), len(test)), (8, 4)) + self.assertFalse(set(train.id) & set(test.id)) + self.assertNotIn("answer", train) + self.assertNotIn("assets", train) + self.assertTrue(train.model_0_cost.between(0, 1).all()) + for name in ("model_0_performance", "model_1_performance", "model_0_raw_cost", "model_1_output_is_actual"): + self.assertIn(name, train) + + def test_named_alignment_frozen_costs_and_all_zero_queries(self): + train, test, _ = download_dataset(vl_args(self.root)) + row = train.set_index("id").loc["RealWorldQA/5"] + self.assertEqual((row.model_0_performance, row.model_1_performance), (1, 0)) + self.assertAlmostEqual(row.model_0_raw_cost, (281+5)*0.05/1e6) + self.assertAlmostEqual(row.model_1_raw_cost, (281*0.18+8*0.25)/1e6) + zero = train.set_index("id").loc["RealWorldQA/0"] + self.assertEqual((zero.model_0_performance, zero.model_1_performance), (0, 0)) + self.assertFalse(train.set_index("id").loc["RealWorldQA/4"].model_1_output_is_actual) + self.assertFalse(any("response" in c for c in train.columns)) + protocol = train.attrs["orbit_protocol"] + self.assertEqual(protocol["excluded_dev_queries"], 2) + self.assertEqual(protocol["estimated_output_records"], 1) + self.assertIn("estimated", protocol["cost_scope"]) + self.assertTrue(all(len(item["sha256"]) == 64 for item in protocol["sources"].values())) + + def test_duplicate_score_output_input_identities_fail(self): + for kind in ("score", "output", "input"): + with self.subTest(kind=kind): + write_vl_fixture(self.root) + if kind == "score": + path = self.root / "ORACLE/score/realworldqa.parquet" + frame = pd.read_parquet(path) + pd.concat([frame, frame.iloc[:1]]).to_parquet(path, index=False) + else: + name = "output_tokens_per_model" if kind == "output" else "token_stats_per_sample" + self._csv(name, lambda df: pd.concat([df, df.iloc[:1]])) + with self.assertRaisesRegex(ValueError, "duplicate identities"): + download_dataset(vl_args(self.root)) + + def test_missing_costs_error_or_explicit_common_support(self): + self._csv("output_tokens_per_model", lambda df: df.iloc[1:]) + with self.assertRaisesRegex(ValueError, "missing/failed"): + download_dataset(vl_args(self.root)) + train, test, _ = download_dataset(vl_args(self.root, missing_policy="drop-query")) + self.assertEqual(train.attrs["orbit_protocol"]["coverage"][0]["dropped_queries"], 1) + + def test_invalid_tokens_and_actual_flags_fail(self): + for column, value in (("output_tokens", -1), ("output_tokens", 1.5), ("output_tokens", np.inf), ("is_actual", "perhaps")): + with self.subTest(column=column, value=value): + write_vl_fixture(self.root) + def edit(df): + df[column] = df[column].astype(object) + df.loc[0, column] = value + return df + self._csv("output_tokens_per_model", edit) + with self.assertRaises(ValueError): + download_dataset(vl_args(self.root)) + + def test_input_total_mismatch_and_invalid_scores_fail(self): + self._csv("token_stats_per_sample", lambda df: df.assign(total_input_tokens=df.total_input_tokens+1)) + with self.assertRaisesRegex(ValueError, "must equal"): + download_dataset(vl_args(self.root)) + write_vl_fixture(self.root) + path = self.root / "ORACLE/score/realworldqa.parquet" + frame = pd.read_parquet(path) + frame["quality_raw"] = frame.quality_raw.astype(float) + frame.loc[0, "quality_raw"] = 1.5 + frame.to_parquet(path, index=False) + with self.assertRaisesRegex(ValueError, "invalid scores"): + download_dataset(vl_args(self.root)) + + def test_estimated_output_policies(self): + with self.assertRaisesRegex(ValueError, "Estimated output"): + download_dataset(vl_args(self.root, estimated_output_policy="error")) + train, test, _ = download_dataset(vl_args(self.root, estimated_output_policy="drop-query", missing_policy="drop-query")) + self.assertNotIn("RealWorldQA/4", train.id.tolist()+test.id.tolist()) + + def test_official_split_duplicates_and_missing_assignments_fail(self): + path = self.root / "SPLITS/test.jsonl" + original = path.read_text() + path.write_text(original+json.dumps({"sample_id": "RealWorldQA/0"})+"\n") + with self.assertRaisesRegex(ValueError, "Official splits"): + download_dataset(vl_args(self.root)) + path.write_text("\n".join(original.splitlines()[1:])+"\n") + with self.assertRaisesRegex(ValueError, "lack official"): + download_dataset(vl_args(self.root)) + + def test_official_overlap_with_dev_is_checked_before_subsampling(self): + path = self.root / "BENCHMARKS/vqa_oe/realworldqa_samples.jsonl" + records = [json.loads(line) for line in path.read_text().splitlines()] + records[9]["prompt"] = records[0]["prompt"] + records[9]["assets"] = records[0]["assets"] + path.write_text("".join(json.dumps(r)+"\n" for r in records)) + with self.assertRaisesRegex(ValueError, "Identical router inputs"): + download_dataset(vl_args(self.root, max_samples=2)) + + def test_tsv_relocation_reference_and_multiframe_contact_sheet(self): + train, test, _ = download_dataset(vl_args(self.root, modality="text+image", image_cell_size=32)) + rows = train.set_index("id") + self.assertEqual(rows.loc["RealWorldQA/0"].image_path, rows.loc["RealWorldQA/1"].image_path) + with Image.open(rows.loc["RealWorldQA/0"].image_path) as image: + self.assertEqual(image.getpixel((0, 0)), (255, 0, 0)) + with Image.open(rows.loc["RealWorldQA/2"].image_path) as image: + self.assertEqual(image.size, (64, 32)) + self.assertEqual(len(train.attrs["orbit_protocol"]["selected_image_sha256"]), 2) + + def test_text_only_identity_does_not_include_invisible_images(self): + path = self.root / "BENCHMARKS/vqa_oe/realworldqa_samples.jsonl" + records = [json.loads(line) for line in path.read_text().splitlines()] + records[9]["prompt"] = records[0]["prompt"] + records[9]["assets"][0]["index"] = 2 # Distinct visible image(s). + path.write_text("".join(json.dumps(r)+"\n" for r in records)) + with self.assertRaisesRegex(ValueError, "Identical router inputs"): + download_dataset(vl_args(self.root, modality="text")) + train, test, _ = download_dataset(vl_args(self.root, modality="text", text_overlap_policy="drop-query")) + self.assertNotIn("RealWorldQA/0", train.id.tolist()+test.id.tolist()) + self.assertEqual(train.attrs["orbit_protocol"]["text_overlap_dropped_queries"], 2) + train, test, _ = download_dataset(vl_args(self.root, modality="text+image")) + self.assertIn("RealWorldQA/0", train.id.tolist()) + train, test, _ = download_dataset(vl_args(self.root, modality="text+image", text_overlap_policy="drop-query")) + self.assertNotIn("RealWorldQA/0", train.id.tolist()) + self.assertEqual(train.attrs["orbit_protocol"]["text_overlap_dropped_queries"], 2) + + def test_materialized_jpeg_preserves_original_encoded_bytes(self): + buffer = io.BytesIO() + Image.new("RGB", (20, 16), "red").save(buffer, format="JPEG") + raw = buffer.getvalue() + loader = VLRouterBenchLoader(str(self.root), vl_args(self.root)) + path = loader._decode_images(base64.b64encode(raw).decode())[0] + self.assertEqual(path.read_bytes(), raw) + with Image.open(path) as image: + self.assertEqual(image.format, "JPEG") + + def test_tsv_missing_cyclic_or_invalid_references_fail(self): + path = self.root / "vlm_router_data/TSV_images/RealWorldQA.tsv" + original = path.read_text() + for images in (["20", "10", "20"], ["999", "999", "999"], ["!!invalid!!"]*3): + with self.subTest(images=images): + pd.DataFrame({"index": [10, 20, 30], "image": images}).to_csv(path, sep="\t", index=False) + with self.assertRaises(ValueError): + download_dataset(vl_args(self.root, modality="text+image")) + path.write_text(original) + + def test_missing_assets_are_not_silent_text_only_evaluation(self): + (self.root / "vlm_router_data/TSV_images/RealWorldQA.tsv").unlink() + with self.assertRaises(FileNotFoundError): + download_dataset(vl_args(self.root, modality="text+image")) + # Text-only is an explicit ablation and needs no image decoder/resources. + train, test, _ = download_dataset(vl_args(self.root, modality="text")) + self.assertNotIn("image_path", train) + + def test_image_row_maps_are_reused_between_router_loads(self): + args = vl_args(self.root, modality="text+image") + first, _, _ = download_dataset(args) + with patch("utils.vl_routerbench.csv.DictReader", side_effect=AssertionError("Do not rescan TSV")): + second, _, _ = download_dataset(args) + self.assertEqual(first.image_path.tolist(), second.image_path.tolist()) + + def test_relative_direct_image_paths(self): + image_root = self.root / "vlm_router_data/TSV_images" + Image.new("RGB", (18, 14), "green").save(image_root / "direct.png") + path = self.root / "BENCHMARKS/vqa_oe/realworldqa_samples.jsonl" + records = [json.loads(line) for line in path.read_text().splitlines()] + records[0]["assets"] = [{"type": "image", "uri": "direct.png"}] + path.write_text("".join(json.dumps(r)+"\n" for r in records)) + train, _, _ = download_dataset(vl_args(self.root, modality="text+image")) + self.assertEqual(train.set_index("id").loc["RealWorldQA/0"].image_path, str(image_root / "direct.png")) + + def test_no_pickle_or_matrix_reading(self): + with patch("pickle.load", side_effect=AssertionError("No pickle")), patch("numpy.load", side_effect=AssertionError("No matrix")): + download_dataset(vl_args(self.root)) + + def test_unknown_model_or_benchmark_fails(self): + for options in ({"models": ["unknown"]}, {"benchmarks": ["unknown"]}): + args = vl_args(self.root) + args["dataset"].update(options) + with self.assertRaises(ValueError): + download_dataset(args) + + def test_missing_prepared_export_has_actionable_error(self): + args = vl_args(self.root / "not-prepared") + with self.assertRaisesRegex(FileNotFoundError, "Steps 1 and 2"): + download_dataset(args) + + def test_missing_response_supervision_is_not_fabricated(self): + with self.assertRaisesRegex(ValueError, "response strings"): + download_dataset(vl_args(self.root, include_responses=True)) + + def test_random_split_uses_existing_protocol(self): + train, test, _ = download_dataset(vl_args(self.root, split={"mode": "in-domain", "ratios": {"train": 0.5, "test": 0.5}})) + self.assertEqual((len(train), len(test)), (7, 7)) + self.assertFalse(set(train.input_group) & set(test.input_group)) + + def test_image_path_traversal_invalid_index_and_asset_types_fail(self): + path = self.root / "BENCHMARKS/vqa_oe/realworldqa_samples.jsonl" + original = path.read_text() + for asset in ( + {"type": "image_tsv", "tsv_file": "../RealWorldQA.tsv", "index": 0}, + {"type": "image_tsv", "tsv_file": "RealWorldQA.tsv", "index": True}, + {"type": "image_tsv", "tsv_file": "", "index": 0}, + {"type": "image", "uri": "../secret.png"}, + {"type": "image_url", "uri": "https://example.com/image.png"}, + ): + with self.subTest(asset=asset): + records = [json.loads(line) for line in original.splitlines()] + records[0]["assets"] = [asset] + path.write_text("".join(json.dumps(r)+"\n" for r in records)) + with self.assertRaises(ValueError): + download_dataset(vl_args(self.root)) + + def test_bounded_multimodal_only_decodes_retained_splits(self): + path = self.root / "BENCHMARKS/vqa_oe/realworldqa_samples.jsonl" + records = [json.loads(line) for line in path.read_text().splitlines()] + # Deliberately unavailable images on excluded dev queries are harmless. + records[8]["assets"][0]["index"] = 10000 + path.write_text("".join(json.dumps(r)+"\n" for r in records)) + train, test, _ = download_dataset(vl_args(self.root, modality="text+image", max_samples=2)) + self.assertEqual((len(train), len(test)), (2, 2)) + + def test_download_helper_refuses_overwrites_and_unverified_archives(self): + from scripts import download_vlrouterbench as helper + with patch("sys.argv", ["download_vlrouterbench", "--output-dir", str(self.root)]), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + helper.main() + archive = self.root / "not-an-official-archive.tar.gz" + archive.write_bytes(b"not the official release") + output = self.root / "safe-output" + with patch("sys.argv", ["download_vlrouterbench", "--output-dir", str(output), "--archive", str(archive)]): + with self.assertRaisesRegex(ValueError, "hash"): + helper.main() + self.assertFalse(any(output.iterdir())) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/data.py b/utils/data.py index 504f7bc..a7005b1 100644 --- a/utils/data.py +++ b/utils/data.py @@ -17,6 +17,7 @@ from utils.benchmark_adapters import ( MMRBenchV2Adapter, R2BenchAdapter, XRouteBenchAdapter, ) +from utils.vl_routerbench import VLRouterBenchAdapter def _safe_archive_target(target_dir: str, member_name: str) -> Path: @@ -1564,6 +1565,10 @@ class R2BenchLoader(R2BenchAdapter, BaseDatasetLoader): """Treat each released (model, nominal budget) as a routing candidate.""" +class VLRouterBenchLoader(VLRouterBenchAdapter, BaseDatasetLoader): + """Vision-language outcomes exported by the official VL-RouterBench pipeline.""" + + # Registry to allow users to add new dataset loaders _LOADER_REGISTRY: Dict[str, Type[BaseDatasetLoader]] = { RouterbenchLoader.name: RouterbenchLoader, @@ -1574,6 +1579,7 @@ class R2BenchLoader(R2BenchAdapter, BaseDatasetLoader): XRouteBenchLoader.name: XRouteBenchLoader, MMRBenchV2Loader.name: MMRBenchV2Loader, R2BenchLoader.name: R2BenchLoader, + VLRouterBenchLoader.name: VLRouterBenchLoader, } diff --git a/utils/embedding.py b/utils/embedding.py index 4fdaeae..99b2ea3 100644 --- a/utils/embedding.py +++ b/utils/embedding.py @@ -47,6 +47,7 @@ def keys(self) -> List[str]: TEXT_EMBEDDERS = EmbedderRegistry("text-embedders") IMAGE_EMBEDDERS = EmbedderRegistry("image-embedders") +JOINT_EMBEDDERS = EmbedderRegistry("joint-embedders") # ========================= @@ -65,6 +66,9 @@ class Embedder: "training": bool "out_dim": int or None optional common: + "joint_model": optional joint encoder (mutually exclusive with text/image_model) + "fusion": "concat" (default) or "aligned_mean" for matched frozen encoders + "text_weight": text weight for aligned_mean, strictly between zero and one "device": str (e.g., "cuda:0") "use_fp16": bool "text_model_dir": str @@ -78,6 +82,22 @@ def __init__(self, args: Dict[str, Any]): self.normalize = bool(em.get("normalize", False)) self.training = bool(em.get("training", False)) self.out_dim = em.get("out_dim", None) + self.fusion = em.get("fusion", "concat") + self.text_weight = float(em.get("text_weight", 0.5)) + if self.fusion not in {"concat", "aligned_mean"}: + raise ValueError("embeddings.fusion must be concat or aligned_mean.") + if not np.isfinite(self.text_weight) or not 0 < self.text_weight < 1: + raise ValueError("embeddings.text_weight must lie strictly between 0 and 1.") + if self.fusion == "aligned_mean" and self.training: + raise ValueError("aligned_mean requires frozen, aligned encoders, not random projections.") + self.joint_embedder = None + self.joint_dim = None + if em.get("joint_model"): + if em.get("text_model") or em.get("image_model") or self.training: + raise ValueError("A joint encoder cannot be combined with separate encoders/projections.") + self.joint_embedder, self.joint_dim = JOINT_EMBEDDERS.create(em["joint_model"], args) + if self.out_dim is not None and self.out_dim != self.joint_dim: + raise ValueError("Joint encoder dimension must equal embeddings.out_dim.") self.text_embedder: Optional[EmbedFn] = None self.image_embedder: Optional[EmbedFn] = None @@ -89,6 +109,10 @@ def __init__(self, args: Dict[str, Any]): if em.get("image_model"): self.image_embedder, self.image_dim = CreateImageEmbedder(em["image_model"], args) + if self.fusion == "aligned_mean" and ( + self.text_dim is None or self.text_dim != self.image_dim + ): + raise ValueError("aligned_mean requires matched text/image encoders of equal dimension.") self.text_proj: Optional[nn.Parameter] = None self.image_proj: Optional[nn.Parameter] = None @@ -173,6 +197,30 @@ def _project(self, emb: torch.Tensor, modality: str) -> torch.Tensor: def run_embed(self, texts=None, images=None) -> torch.Tensor: batch_size = int(self.args["embeddings"].get("batch_size", 256)) + if batch_size < 1: + raise ValueError("Embedding batch_size must be positive.") + if self.joint_embedder is not None: + texts = [texts] if isinstance(texts, str) else (list(texts) if texts is not None else None) + images = list(images) if images is not None else None + total = len(texts) if texts is not None else (len(images) if images is not None else 0) + if texts is not None and images is not None and len(texts) != len(images): + raise ValueError("text and image batch sizes must match.") + if not total: + raise ValueError("Joint embedding requires nonempty inputs.") + batches = [] + for offset in range(0, total, batch_size): + end = min(offset + batch_size, total) + values = torch.as_tensor(self.joint_embedder( + texts=texts[offset:end] if texts is not None else None, + images=images[offset:end] if images is not None else None, + ), dtype=torch.float32) + if values.shape != (end - offset, self.joint_dim) or not torch.isfinite(values).all(): + raise ValueError("Joint encoder returned invalid shape/non-finite vectors.") + norms = torch.linalg.vector_norm(values, dim=1, keepdim=True) + if not torch.isfinite(norms).all() or (norms <= 1e-12).any(): + raise ValueError("Joint encoder returned an invalid/zero vector norm.") + batches.append(values / norms if self.normalize else values) + return torch.cat(batches, dim=0) parts: List[torch.Tensor] = [] N: Optional[int] = None @@ -224,7 +272,19 @@ def run_embed(self, texts=None, images=None) -> torch.Tensor: if len(parts) == 0: raise ValueError("no inputs provided to run_embed") - combined = torch.cat(parts, dim=1) + if self.fusion == "aligned_mean": + # Both encoders must belong to the same pretrained aligned space. + norms = [part.norm(dim=1, keepdim=True) for part in parts] + if any(not torch.isfinite(part).all() or not torch.isfinite(norm).all() + or (norm <= 1e-12).any() for part, norm in zip(parts, norms)): + raise ValueError("Aligned encoders must return finite nonzero vectors for each modality.") + parts = [part / norm for part, norm in zip(parts, norms)] + combined = (self.text_weight * parts[0] + (1 - self.text_weight) * parts[1] + if len(parts) == 2 else parts[0]) + if not torch.isfinite(combined).all() or (combined.norm(dim=1) <= 1e-12).any(): + raise ValueError("Aligned fusion produced invalid/zero vectors.") + else: + combined = torch.cat(parts, dim=1) if self.normalize: norm = torch.linalg.norm(combined, dim=1, keepdim=True).clamp_min(1e-12) @@ -576,6 +636,12 @@ def embed_fn(images: Union[str, os.PathLike, Image.Image, list], batch_size: int # Registration (model name -> builder) # ========================= +@JOINT_EMBEDDERS.register("Qwen/Qwen3-VL-Embedding-8B") +def _build_qwen3_vl(args: Dict[str, Any]): + from utils.qwen_vl_embedding import QwenVLEmbedder + return QwenVLEmbedder(args["embeddings"]), 4096 + + @TEXT_EMBEDDERS.register("Qwen3-Embedding-8B") def _build_qwen3_8b(args: Dict[str, Any]): model_dir = args["embeddings"].get("text_model_dir", "./Qwen3-Embedding-8B") diff --git a/utils/qwen_vl_embedding.py b/utils/qwen_vl_embedding.py new file mode 100644 index 0000000..5a8a522 --- /dev/null +++ b/utils/qwen_vl_embedding.py @@ -0,0 +1,140 @@ +"""Joint Qwen3-VL embeddings; never substitute text-only/image concatenations. + +Local mode uses the model-card SentenceTransformer interface and local weights. +API mode targets an explicitly configured RouteFM-compatible embedding service: +POST /embeddings, input=[{text, image: data-URL}], indexed data[].embedding. +This multimodal input schema is not a universal OpenAI/vLLM API contract. +""" + +from __future__ import annotations + +import base64 +import json +import os +from pathlib import Path +import urllib.error +import urllib.parse +import urllib.request + +import numpy as np +from PIL import Image + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None # Do not forward authorization or images to a different URL. + + +class QwenVLEmbedder: + def __init__(self, embeddings): + self.config = dict(embeddings.get("qwen", {})) + self.backend = self.config.get("backend", "local") + self.instruction = self.config.get("instruction") + self.model_name = self.config.get("model", "Qwen/Qwen3-VL-Embedding-8B") + self.model = None + if self.backend == "local": + model_dir = Path(self.config.get("model_dir", "./Qwen3-VL-Embedding-8B")) + if not model_dir.is_dir(): + raise FileNotFoundError( + "Download Qwen/Qwen3-VL-Embedding-8B first and set embeddings.qwen.model_dir, " + "or explicitly configure the api backend. No substitute encoder is used." + ) + from sentence_transformers import SentenceTransformer + dtype = self.config.get("dtype", "auto") + if dtype not in {"auto", "float32", "float16", "bfloat16"}: + raise ValueError("Qwen local dtype must be auto, float32, float16 or bfloat16.") + self.model = SentenceTransformer( + str(model_dir), device=embeddings.get("device", "cpu"), + local_files_only=True, trust_remote_code=False, + model_kwargs={"torch_dtype": dtype}, + ) + elif self.backend == "api": + self.base_url = str(self.config.get("base_url") or "").rstrip("/") + url = urllib.parse.urlsplit(self.base_url) + if (url.scheme not in {"http", "https"} or not url.hostname or url.username + or url.password or url.query or url.fragment): + raise ValueError("Qwen API base_url must be an explicit HTTP(S) endpoint without credentials/query.") + if url.scheme != "https" and url.hostname not in {"127.0.0.1", "localhost", "::1"}: + raise ValueError("Qwen API requires HTTPS except for a local service.") + self.api_key_env = self.config.get("api_key_env", "ROUTEFM_EMBEDDING_API_KEY") + if not os.environ.get(self.api_key_env): + raise ValueError(f"Set {self.api_key_env} for the explicitly selected Qwen embedding service.") + self.timeout = float(self.config.get("timeout", 60)) + if not np.isfinite(self.timeout) or self.timeout <= 0: + raise ValueError("Qwen API timeout must be finite and positive.") + handlers = [_NoRedirect()] + if url.hostname in {"127.0.0.1", "localhost", "::1"}: + handlers.append(urllib.request.ProxyHandler({})) + self.opener = urllib.request.build_opener(*handlers) + else: + raise ValueError("embeddings.qwen.backend must be local or api.") + + def __call__(self, texts=None, images=None): + if texts is None and images is None: + raise ValueError("Qwen joint encoder requires inputs.") + count = len(texts) if texts is not None else len(images) + if images is not None and len(images) != count: + raise ValueError("Qwen text/image batch sizes must match.") + inputs = [] + for index in range(count): + value = {} + if texts is not None: + value["text"] = str(texts[index]) + if images is not None: + image = images[index] + if not isinstance(image, (str, Path)) or not Path(image).is_file(): + raise ValueError("Qwen image inputs must be existing local files; no silent image omission.") + value["image"] = str(Path(image).resolve()) + inputs.append(value) + values = np.asarray(self._encode_batch(inputs), dtype=np.float32) + if values.shape != (count, 4096) or not np.isfinite(values).all(): + raise ValueError("Qwen3-VL-Embedding-8B must return finite [batch,4096] vectors.") + norms = np.linalg.norm(values.astype(np.float64), axis=1, keepdims=True) + if np.any(norms <= 1e-12): + raise ValueError("Qwen joint encoder returned a zero vector.") + return (values / norms).astype(np.float32) + + def _encode_batch(self, inputs): + if self.backend == "local": + options = {"normalize_embeddings": True, "convert_to_numpy": True, "show_progress_bar": False} + if self.instruction is not None: + options["prompt"] = self.instruction + return self.model.encode(inputs, **options) + + payload_inputs = [] + for item in inputs: + value = dict(item) + if "image" in value: + path = Path(value["image"]) + if path.stat().st_size > 32 * 1024 * 1024: + raise ValueError("Qwen API image exceeds the 32 MiB upload limit.") + with Image.open(path) as image: + mime = Image.MIME.get(image.format) + image.verify() + if mime not in {"image/jpeg", "image/png", "image/webp"}: + raise ValueError("Qwen API supports local JPEG/PNG/WebP images only.") + value["image"] = f"data:{mime};base64," + base64.b64encode(path.read_bytes()).decode("ascii") + if self.instruction is not None: + value["instruction"] = self.instruction + payload_inputs.append(value) + request = urllib.request.Request( + self.base_url + "/embeddings", + data=json.dumps({"model": self.model_name, "input": payload_inputs}).encode(), + headers={"Authorization": "Bearer " + os.environ[self.api_key_env], "Content-Type": "application/json"}, + method="POST", + ) + try: + with self.opener.open(request, timeout=self.timeout) as response: + body = response.read(32 * 1024 * 1024 + 1) + except (urllib.error.URLError, TimeoutError): + # Provider errors can contain credentials/input; do not echo their body. + raise RuntimeError("Qwen embedding request failed; check endpoint/auth/service logs.") from None + if len(body) > 32 * 1024 * 1024: + raise ValueError("Qwen embedding response exceeds the size limit.") + payload = json.loads(body) + rows = payload.get("data", []) + indices = [row.get("index") for row in rows] + if (any(type(index) is not int for index in indices) + or sorted(indices) != list(range(len(inputs)))): + raise ValueError("Qwen embedding response must contain each batch index exactly once.") + return [row["embedding"] for row in sorted(rows, key=lambda row: row["index"])] diff --git a/utils/vl_routerbench.py b/utils/vl_routerbench.py new file mode 100644 index 0000000..0535e79 --- /dev/null +++ b/utils/vl_routerbench.py @@ -0,0 +1,407 @@ +"""Adapt VL-RouterBench's named export tables, never positional/pickle matrices. + +Prepare BENCHMARKS, ORACLE/score, SPLITS and token statistics with the upstream +pipeline first. See docs/vl_routerbench.md. All router interfaces stay unchanged. +""" + +from __future__ import annotations + +import ast +import base64 +import csv +import hashlib +import io +import json +import math +from pathlib import Path + +import numpy as np +import pandas as pd +from PIL import Image + +from utils.benchmark_adapters import ( + FrozenBenchmarkAdapter, METADATA_ROOT, _input_key, _numbers, _relative_file, + _require_columns, _text, +) + + +# Bounded in-process maps avoid rescanning multi-GB TSVs for each router in a +# comparison run. Source stat + requested rows + cache root identify entries. +# No labels, scores or model selections participate in this cache. +_TSV_IMAGE_CACHE = {} + + +def _read_jsonl(path): + with path.open(encoding="utf-8") as handle: + return [json.loads(line) for line in handle if line.strip()] + + +def _unique(frame, columns, location): + if frame.duplicated(columns).any(): + raise ValueError(f"{location}: duplicate identities for {columns}.") + + +def _actual(series): + # In particular, bool('False') must never mark an estimate as observed. + values = series.astype(str).str.lower().str.strip() + if not values.isin(["true", "false", "1", "0"]).all(): + raise ValueError("is_actual must contain explicit boolean values.") + return values.isin(["true", "1"]) + + +class VLRouterBenchAdapter(FrozenBenchmarkAdapter): + name = "VLRouterBench" + hf_id = "KinghtH/VL-RouterBench" + default_target_path = "./data/vl_routerbench" + default_revision = "6e44fbf50e170f8e72734b49f63f2b52a8866377" + + def download(self): + """Locate upstream exports; raw archives are not HF tabular datasets.""" + self.source_root = Path(self.options.get("source_dir", self.target_path)).resolve() + required = ["BENCHMARKS", "ORACLE/score", "reports/token_statistics"] + missing = [p for p in required if not _relative_file(self.source_root, p).is_dir()] + if missing: + raise FileNotFoundError( + f"VLRouterBench requires upstream Steps 1 and 2 under {self.source_root}; " + f"missing {missing}. See docs/vl_routerbench.md. The public archive " + "contains raw artifacts, not a load_dataset-compatible table." + ) + return self.source_root + + def _read(self, root, relative, reader): + path = _relative_file(root, relative) + if not path.is_file(): + raise FileNotFoundError(f"Missing VLRouterBench export: {path}") + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + self.protocol["sources"][relative] = { + "origin": "local upstream export (release provenance not independently verified)", + "sha256": digest.hexdigest(), + } + return reader(path) + + def process(self, source): + if self.options.get("include_responses", False): + raise ValueError("Standard VLRouterBench token exports do not contain response strings; set include_responses=false.") + root = Path(source).resolve() + self.source_root = root + with (METADATA_ROOT / "vlrouterbench.json").open(encoding="utf-8") as handle: + registry = json.load(handle) + models = self._models(registry["models"]) + tasks = self.options.get("benchmarks", registry["benchmarks"]) + if (not isinstance(tasks, list) or not tasks or len(set(tasks)) != len(tasks) + or set(tasks) - set(registry["benchmarks"])): + raise ValueError("dataset.benchmarks must select distinct VLRouterBench dataset IDs.") + tables = [] + scores = [] + for task in sorted(tasks): + task_type = registry["benchmarks"][task] + relative = f"BENCHMARKS/{task_type}/{task.lower()}_samples.jsonl" + samples = pd.DataFrame(self._read(root, relative, _read_jsonl)) + _require_columns(samples, ["sample_id", "dataset", "prompt", "assets"], relative) + samples["sample_id"] = _text(samples.sample_id, "sample_id") + samples["prompt"] = _text(samples.prompt, "prompt") + if not samples.dataset.eq(task).all(): + raise ValueError(f"{relative}: conflicting dataset identity.") + _unique(samples, ["sample_id"], relative) + samples = samples[["sample_id", "prompt", "assets"]].copy() + samples["eval_name"] = task + tables.append(samples) + relative = f"ORACLE/score/{task.lower()}.parquet" + quality = self._read(root, relative, pd.read_parquet) + _require_columns(quality, ["sample_id", "model_id", "quality_raw"], relative) + quality["sample_id"] = _text(quality.sample_id, "score sample_id") + quality["model_id"] = _text(quality.model_id, "score model_id") + quality = quality[quality.model_id.isin(models)].copy() + _unique(quality, ["sample_id", "model_id"], relative) + if set(quality.sample_id) - set(samples.sample_id): + raise ValueError(f"{relative}: scores refer to unknown samples.") + quality["quality_raw"] = _numbers(quality.quality_raw, "quality_raw", score=True) + scores.append(quality[["sample_id", "model_id", "quality_raw"]]) + frame = pd.concat(tables, ignore_index=True) + _unique(frame, ["sample_id"], "BENCHMARKS") + quality = pd.concat(scores, ignore_index=True) + _unique(quality, ["sample_id", "model_id"], "ORACLE/score") + + input_tokens = self._read(root, "reports/token_statistics/token_stats_per_sample.csv", pd.read_csv) + outputs = self._read(root, "reports/token_statistics/output_tokens_per_model.csv", pd.read_csv) + _require_columns(input_tokens, ["sample_id", "text_tokens", "image_tokens", "total_input_tokens"], "input tokens") + _require_columns(outputs, ["sample_id", "model", "output_tokens", "is_actual"], "output tokens") + input_tokens["sample_id"] = _text(input_tokens.sample_id, "input sample_id") + outputs["sample_id"] = _text(outputs.sample_id, "output sample_id") + outputs["model"] = _text(outputs.model, "output model") + input_tokens = input_tokens[input_tokens.sample_id.isin(frame.sample_id)].copy() + outputs = outputs[outputs.sample_id.isin(frame.sample_id) & outputs.model.isin(models)].copy() + _unique(input_tokens, ["sample_id"], "input tokens") + _unique(outputs, ["sample_id", "model"], "output tokens") + for column in ("text_tokens", "image_tokens", "total_input_tokens"): + input_tokens[column] = _numbers(input_tokens[column], column, integer=True) + if not np.array_equal(input_tokens.total_input_tokens, input_tokens.text_tokens + input_tokens.image_tokens): + raise ValueError("total_input_tokens must equal text_tokens + image_tokens.") + outputs["output_tokens"] = _numbers(outputs.output_tokens, "output_tokens", integer=True) + outputs["is_actual"] = _actual(outputs.is_actual) + policy = self.options.get("estimated_output_policy", "allow") + if policy not in {"allow", "error", "drop-query"}: + raise ValueError("estimated_output_policy must be allow, error or drop-query.") + estimated = outputs[~outputs.is_actual] + if policy == "error" and not estimated.empty: + raise ValueError("Estimated output token records are present.") + self.protocol["estimated_output_records"] = len(estimated) + self.protocol["estimated_output_policy"] = policy + if policy == "drop-query": + outputs = outputs[outputs.is_actual].copy() + outputs = outputs.merge(input_tokens[["sample_id", "total_input_tokens"]], on="sample_id", how="left", validate="many_to_one") + prices = registry["models"] + for index, model in enumerate(models): + score = quality[quality.model_id == model].set_index("sample_id").quality_raw + usage = outputs[outputs.model == model].set_index("sample_id") + price = prices[model] + cost = (usage.total_input_tokens * price["input_price_per_1m"] + usage.output_tokens * price["output_price_per_1m"]) / 1e6 + frame[f"model_{index}_performance"] = frame.sample_id.map(score) + frame[f"model_{index}_cost"] = frame.sample_id.map(cost) + frame[f"model_{index}_input_tokens"] = frame.sample_id.map(usage.total_input_tokens) + frame[f"model_{index}_output_tokens"] = frame.sample_id.map(usage.output_tokens) + frame[f"model_{index}_output_is_actual"] = frame.sample_id.map(usage.is_actual) + frame = frame.rename(columns={"sample_id": "id"}) + frame["image_refs"] = frame.assets.map(self._asset_identity) + frame = self._complete(frame, models, default_policy="error") + text_only = "image" not in self.args["modality"].split("+") + if text_only: + # Blind routers cannot distinguish identical questions by image ID. + frame["input_group"] = frame.prompt.map(lambda prompt: _input_key({"prompt": prompt})) + + mode = self.options["split"]["mode"] + if mode == "official": + assignments = {} + for split in ("train", "dev", "test"): + records = self._read(root, f"SPLITS/{split}.jsonl", _read_jsonl) + for item in records: + sid = item.get("sample_id") + if not isinstance(sid, str) or not sid.strip() or sid in assignments: + raise ValueError("Official splits contain duplicate/invalid sample IDs.") + assignments[sid] = split + frame["source_split"] = frame.id.map(assignments) + if frame.source_split.isna().any(): + raise ValueError("Samples lack official train/dev/test assignment.") + # Dev never becomes training supervision or test data. Check overlap + # before max_samples can hide it, including train/dev and dev/test. + overlap_policy = self.options.get("text_overlap_policy", "error") + if overlap_policy not in {"error", "drop-query"}: + raise ValueError("text_overlap_policy must be error or drop-query.") + if overlap_policy == "drop-query": + # Apply the same explicit population filter to image routers, + # too, when comparing them against blind text-only methods. + text_groups = frame.prompt.map(lambda prompt: _input_key({"prompt": prompt})) + text_counts = frame.groupby(text_groups).source_split.nunique() + overlapping_groups = text_counts[text_counts.gt(1)].index + mask = text_groups.isin(overlapping_groups) + dropped = frame[mask] + self.protocol["text_overlap_dropped_by_split"] = dropped.source_split.value_counts().to_dict() + self.protocol["text_overlap_dropped_queries"] = len(dropped) + self.protocol["text_overlap_groups"] = len(overlapping_groups) + frame = frame[~mask].copy() + if frame.groupby("input_group").source_split.nunique().gt(1).any(): + raise ValueError("Identical router inputs occur across official splits; text-only views may set text_overlap_policy=drop-query or use grouped in-domain splits.") + self.protocol["text_overlap_policy"] = overlap_policy + self.protocol["excluded_dev_queries"] = int(frame.source_split.eq("dev").sum()) + self.protocol["split_source"] = "local upstream train/dev/test export files; original paper partitions not independently certified" + # Decode images only after filtering and optional bounded subsampling. + self.protocol.update({ + "benchmarks": sorted(tasks), "raw_cost_unit": "USD", + "prices": {m: prices[m] for m in models}, + "price_source": registry["pricing_source"], + "cost_scope": "frozen reference prices times upstream token counts; input/image tokens estimated, output tokens counted or explicitly estimated; not actual provider bills", + "output_actual_semantics": "is_actual means upstream counted a released response instead of using a length fallback; it does not certify provider usage or per-model tokenization", + "quality_scope": "upstream quality_raw in [0,1]; all-zero samples retained; no trainable-ID filtering", + "input_representation": "upstream prompt plus assets for image modality, prompt only for text ablations; no answers, responses, correctness or costs in router inputs", + "include_responses": False, + "multi_image": self.options.get("multi_image", "contact-sheet"), + "image_cell_size": int(self.options.get("image_cell_size", 512)), + }) + return models, frame + + def _asset_identity(self, assets): + if not isinstance(assets, list) or not assets: + raise ValueError("Every VLRouterBench query requires nonempty image assets.") + refs = [] + for asset in assets: + if not isinstance(asset, dict): + raise ValueError("Image assets must be dictionaries.") + kind = asset.get("type") + if kind == "image_tsv": + name = str(asset.get("tsv_file", "")).replace("\\", "/") + if not name.strip() or ".." in Path(name).parts: + raise ValueError("Unsafe or empty TSV image path.") + index = asset.get("index") + if isinstance(index, bool) or not isinstance(index, int) or index < 0: + raise ValueError("image_tsv.index must be a non-negative row number.") + refs.append([kind, Path(name).name, index]) + elif kind == "image": + name = str(asset.get("uri", asset.get("path", ""))) + if not name or ".." in Path(name.replace("\\", "/")).parts: + raise ValueError("image assets require a path/uri.") + refs.append([kind, name]) + else: + raise ValueError(f"Unsupported image asset type: {kind!r}") + return refs + + def split(self, frame): + train, test = super().split(frame) + if "image" in self.args["modality"].split("+"): + selected = pd.concat([train, test], ignore_index=True) + self._prepare_tsv_images(selected.assets) + self._image_hashes = set() + for part in (train, test): + part["image_path"] = part.assets.map(self._images) + self.protocol["selected_image_sha256"] = sorted(self._image_hashes) + return train.drop(columns="assets"), test.drop(columns="assets") + + def _tsv_path(self, name): + # Upstream exports commonly embed the author's absolute LMUData path. + # Relocate the *basename* exclusively under the configured trusted root. + normalized = str(name).replace("\\", "/") + if not normalized or ".." in Path(normalized).parts: + raise ValueError("Unsafe TSV image path.") + root = self.options.get("image_root", str(self.source_root / "vlm_router_data/TSV_images")) + path = _relative_file(root, Path(normalized).name) + if not path.is_file(): + raise FileNotFoundError(f"Missing TSV image asset: {path}; set dataset.image_root.") + return path + + def _prepare_tsv_images(self, asset_lists): + previous_limit = csv.field_size_limit(64 * 1024 * 1024) + try: + self._load_tsv_images(asset_lists) + finally: + csv.field_size_limit(previous_limit) + + def _load_tsv_images(self, asset_lists): + requested = {} + self._tsv_images = {} + for assets in asset_lists: + for asset in assets: + if asset["type"] == "image_tsv": + path = self._tsv_path(asset["tsv_file"]) + requested.setdefault(path, set()).add(asset["index"]) + for path, positions in requested.items(): + stat = path.stat() + cache_key = (str(path), stat.st_size, stat.st_mtime_ns, + str(Path(self.target_path).resolve()), tuple(sorted(positions))) + cached = _TSV_IMAGE_CACHE.get(cache_key) + if cached and any(not p.resolve().is_relative_to(Path(self.target_path).resolve()) + for paths in cached.values() for p in paths): + raise ValueError("Decoded image cache escapes its configured root.") + if cached and all(p.is_file() for paths in cached.values() for p in paths): + self._tsv_images.update(cached) + continue + rows = {} + references = {} + with path.open(encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle, delimiter="\t") + if not reader.fieldnames or "image" not in reader.fieldnames: + raise ValueError(f"TSV lacks image column: {path}") + for ordinal, row in enumerate(reader): + key = str(row.get("index", ordinal)) + value = row["image"] + if len(value) <= 64: + references[key] = value + if ordinal in positions: + rows[ordinal] = value + if set(rows) != positions: + raise ValueError(f"TSV row reference is out of bounds: {path}") + needed = {value for value in rows.values() if len(value) <= 64} + seen = set() + while needed: + if needed & seen: + raise ValueError(f"Cyclic TSV image reference: {path}") + seen.update(needed) + # Only materialize referenced base64 values, not the entire TSV. + found = {} + with path.open(encoding="utf-8", newline="") as handle: + for ordinal, row in enumerate(csv.DictReader(handle, delimiter="\t")): + key = str(row.get("index", ordinal)) + if key in needed: + if key in found: + raise ValueError(f"Ambiguous TSV index: {key}") + found[key] = row["image"] + if set(found) != needed: + raise ValueError(f"Unresolved TSV image reference: {path}") + references.update(found) + needed = {value for value in found.values() if len(value) <= 64} + for ordinal, value in rows.items(): + while len(value) <= 64: + value = references[value] + self._tsv_images[(str(path), ordinal)] = self._decode_images(value) + _TSV_IMAGE_CACHE[cache_key] = { + (str(path), ordinal): self._tsv_images[(str(path), ordinal)] + for ordinal in positions + } + if len(_TSV_IMAGE_CACHE) > 16: + del _TSV_IMAGE_CACHE[next(iter(_TSV_IMAGE_CACHE))] + + def _decode_images(self, value): + if value.startswith("["): + value = ast.literal_eval(value) + else: + value = [value] + if not isinstance(value, (list, tuple)) or not value: + raise ValueError("TSV image list is empty or invalid.") + paths = [] + for encoded in value: + if not isinstance(encoded, str): + raise ValueError("TSV images must be base64 strings.") + raw = base64.b64decode(encoded, validate=True) + digest = hashlib.sha256(raw).hexdigest() + # Keep the original JPEG/PNG/etc. bytes: re-encoding JPEGs as PNG + # can expand a full visual release by tens of GB. PIL detects the + # format from the contents and ORBIT's embedder converts to RGB. + path = _relative_file(self.target_path, f"decoded_images/{digest}.image") + if not path.is_file(): + with Image.open(io.BytesIO(raw)) as image: + image.verify() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(raw) + paths.append(path) + return paths + + def _images(self, assets): + paths = [] + for asset in assets: + if asset["type"] == "image_tsv": + path = self._tsv_path(asset["tsv_file"]) + paths.extend(self._tsv_images[(str(path), asset["index"])]) + else: + root = self.options.get("image_root", str(self.source_root / "vlm_router_data/TSV_images")) + name = str(asset.get("uri", asset.get("path", ""))).replace("\\", "/") + if Path(name).is_absolute(): + raise ValueError("Absolute image paths need an explicit upstream export repair; only TSV basenames are relocated.") + path = _relative_file(root, name) + if not path.is_file(): + raise FileNotFoundError(f"Missing image asset: {path}") + paths.append(path) + image_digests = [hashlib.sha256(path.read_bytes()).digest() for path in paths] + self._image_hashes.update(digest.hex() for digest in image_digests) + if len(paths) == 1: + return str(paths[0]) + if self.options.get("multi_image", "contact-sheet") != "contact-sheet": + raise ValueError("Multi-image queries require multi_image=contact-sheet.") + size = int(self.options.get("image_cell_size", 512)) + if not 16 <= size <= 2048: + raise ValueError("image_cell_size must be between 16 and 2048.") + digest = hashlib.sha256(f"vl-contact-sheet:{size}".encode()) + for image_digest in image_digests: + digest.update(image_digest) + destination = _relative_file(self.target_path, f"contact_sheets/{digest.hexdigest()}.png") + if not destination.is_file(): + columns = math.ceil(math.sqrt(len(paths))) + sheet = Image.new("RGB", (columns * size, math.ceil(len(paths) / columns) * size), "white") + for index, path in enumerate(paths): + with Image.open(path) as image: + image = image.convert("RGB") + image.thumbnail((size, size), Image.Resampling.LANCZOS) + sheet.paste(image, ((index % columns) * size, (index // columns) * size)) + destination.parent.mkdir(parents=True, exist_ok=True) + sheet.save(destination) + return str(destination)