diff --git a/ods/extensions/services/dashboard-api/performance_oracle.py b/ods/extensions/services/dashboard-api/performance_oracle.py index ffd1704e93..f7a764b6ef 100644 --- a/ods/extensions/services/dashboard-api/performance_oracle.py +++ b/ods/extensions/services/dashboard-api/performance_oracle.py @@ -873,7 +873,7 @@ def _usable_model_memory_gb(gpu_info: Optional[GPUInfo], system_ram_gb: int | No return max(ram_gb * 0.55, 2.0) if backend in {"cpu", "none", "unknown"} or total_gb <= 0: ram_gb = system_ram_gb if system_ram_gb is not None else _system_ram_gb() - return min(max(ram_gb * 0.35, 3.0), 8.0) + return min(max(ram_gb, 0.0) * 0.35, 8.0) return total_gb diff --git a/ods/extensions/services/dashboard-api/tests/test_model_memory.py b/ods/extensions/services/dashboard-api/tests/test_model_memory.py index fec67d216d..1c6f567c45 100644 --- a/ods/extensions/services/dashboard-api/tests/test_model_memory.py +++ b/ods/extensions/services/dashboard-api/tests/test_model_memory.py @@ -81,7 +81,8 @@ def test_filename_scale_lowers_the_kv_estimate(self): class TestArchitectureAwareKvCache: @pytest.mark.parametrize("backend,kind,ram,vram", [ - ("cpu", "discrete", 8, 0), ("cpu", "discrete", 32, 0), + ("cpu", "discrete", 4, 0), ("cpu", "discrete", 8, 0), + ("cpu", "discrete", 32, 0), ("amd", "unified", 16, 8192), ("amd", "unified", 64, 32768), ("nvidia", "discrete", 8, 24576), ]) diff --git a/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py b/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py index 72080ba615..21cb580982 100644 --- a/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py +++ b/ods/extensions/services/dashboard-api/tests/test_performance_oracle.py @@ -1541,6 +1541,28 @@ def test_pre_download_ranker_prefers_capable_8gb_model_over_bootstrap(data_dir): assert ranked[0]["id"] == "qwen3.5-9b-q4" +def test_pre_download_ranker_does_not_recommend_three_gb_model_on_4gb_cpu(data_dir): + model = { + "id": "qwen3.5-2b-q4", + "name": "Qwen 3.5 2B", + "family": "qwen", + "gguf_file": "Qwen3.5-2B-Q4_K_M.gguf", + "gguf_url": "https://example.invalid/qwen3.5-2b.gguf", + "size_mb": 1281, + "vram_required_gb": 3, + "context_length": 65536, + "specialty": "Fast", + "llm_model_name": "qwen3.5-2b", + } + + ranked = rank_pre_download_models( + [model], _gpu(name="CPU", total_mb=0, backend="cpu"), + profile="qwen", installable_only=True, system_ram_gb=4, + ) + + assert ranked == [] + + def test_pre_download_ranker_accounts_for_long_context_kv_on_4gb_gpu(data_dir, tmp_path): catalog = [ { diff --git a/ods/scripts/select-model.py b/ods/scripts/select-model.py index 1acab482ee..58dbe7ae4c 100644 --- a/ods/scripts/select-model.py +++ b/ods/scripts/select-model.py @@ -168,7 +168,9 @@ def usable_memory_gb(backend: str, memory_type: str, vram_mb: int, ram_gb: int) # Macs/APUs are not handed a model that technically fits but thrashes. return max(float(ram_gb) * 0.55, 2.0), "unified system memory" if backend_key in {"cpu", "none", "unknown"} or vram_mb <= 0: - return min(max(float(ram_gb) * 0.35, 3.0), 8.0), "system RAM" + # Do not floor the model budget above available headroom on small + # hosts. At 4 GB, a 3 GB floor leaves only 1 GB for Linux and ODS. + return min(max(float(ram_gb), 0.0) * 0.35, 8.0), "system RAM" return float(vram_mb) / 1024.0, "GPU VRAM" diff --git a/ods/tests/test_model_selector_cpu_memory_budget.py b/ods/tests/test_model_selector_cpu_memory_budget.py new file mode 100644 index 0000000000..c6533a5f34 --- /dev/null +++ b/ods/tests/test_model_selector_cpu_memory_budget.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Exercise the production CPU model selector at Tier 0 memory limits.""" + +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SELECTOR = ROOT / "scripts" / "select-model.py" +CATALOG = ROOT / "config" / "model-library.json" + + +def run_selector(ram_gb: int) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(SELECTOR), + "--catalog", + str(CATALOG), + "--backend", + "cpu", + "--memory-type", + "none", + "--vram-mb", + "0", + "--ram-gb", + str(ram_gb), + "--profile", + "qwen", + "--tier", + "0", + "--max-size-mb", + "0", + "--host-arch", + "amd64", + "--installable-only", + "--env", + ], + capture_output=True, + text=True, + check=False, + ) + + +def test_four_gb_cpu_host_does_not_get_a_three_gb_model_recommendation(): + result = run_selector(4) + assert result.returncode == 2, result.stdout + assert "no installable model fits" in result.stderr + + +def test_eight_gb_cpu_host_keeps_supported_model_recommendation(): + result = run_selector(8) + assert result.returncode == 0, result.stderr + assert 'LLM_MODEL="qwen3.5-2b"' in result.stdout + assert 'MAX_CONTEXT="65536"' in result.stdout + + +if __name__ == "__main__": + test_four_gb_cpu_host_does_not_get_a_three_gb_model_recommendation() + test_eight_gb_cpu_host_keeps_supported_model_recommendation() + print("CPU model memory budget tests passed: 2")