From 1b9ac0d7709f8cf8a22c0da731726ec63651e6d4 Mon Sep 17 00:00:00 2001 From: Glenn Neuber Date: Wed, 14 Oct 2026 00:42:43 +1000 Subject: [PATCH] preflight: cuda-0-40-2 and mlx-cuda-0-40-2 admit the v0.40.3 lineage v0.40.3 (#488) is a Go-only fold: llama.cpp b11351 and MLX a59cc231 stay, so ADR 0032 lets its builds share the 0.40.2 profiles. Both patterns widen from 0\.40\.2 to 0\.40\.[23]; 0.40.4 stays refused (its rc1 moves llama.cpp). The fold's head, 0.40.2-dynres.1-6-g9f3baaf, passed both on this host's production-like canary (--gpus all): cuda PASS=20 SKIP=9, mlx-cuda PASS=11 SKIP=18 with 0 cudaMemcpy errors. Those runs are added under runs/. TestCudaProfilesForTheV0402Fold: production's 377f9d1 stamp, the fold's head and a v0.40.3 tag stamp move to ADMIT; a dirty v0.40.3 tree and 0.40.4 to REJECT; the resolver check covers a v0.40.3 stamp. Co-Authored-By: Claude Opus 5.5 --- .../vision-suite/preflight/expectations.toml | 10 +- ...uda-0403-fold-canary-gpusall-g9f3baaf.json | 472 ++++++++++++++++++ ...uda-0403-fold-canary-gpusall-g9f3baaf.json | 394 +++++++++++++++ .../vision-suite/preflight/test_verdicts.py | 9 +- 4 files changed, 881 insertions(+), 4 deletions(-) create mode 100644 docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-fold-canary-gpusall-g9f3baaf.json create mode 100644 docs/maxusai/vision-suite/preflight/runs/preflight-mlx-cuda-0403-fold-canary-gpusall-g9f3baaf.json diff --git a/docs/maxusai/vision-suite/preflight/expectations.toml b/docs/maxusai/vision-suite/preflight/expectations.toml index 0609a867354..60d57ea300f 100644 --- a/docs/maxusai/vision-suite/preflight/expectations.toml +++ b/docs/maxusai/vision-suite/preflight/expectations.toml @@ -3587,7 +3587,11 @@ model = "qwen2.5vl:3b-q4_K_M" # payload_proof on that regression. # - 903 here is the tag's. Builds of main after #462 carry #462's 903 (#29953 as merged plus the y-tile guard) # under the same label; it changes the MMQ src1/ids_dst padding, not image token accounting. -version_pattern = '^0\.40\.2-dynres(\.\d+)?-\d+-g[0-9a-f]{7,40}$' +# Widened 2026-10-14 to 0.40.3 for the v0.40.3 fold (#488), as ADR 0032 says for a Go-only fold: v0.40.3 moves no +# native input (llama.cpp b11351 and MLX a59cc231 stay), so its builds share this payload and this profile. Its head, +# 0.40.2-dynres.1-6-g9f3baaf, passed here on a production-like canary (--gpus all): PASS=20 SKIP=9. 0.40.4 stays +# refused: its rc1 moves llama.cpp to b11521. +version_pattern = '^0\.40\.[23]-dynres(\.\d+)?-\d+-g[0-9a-f]{7,40}$' patchset = ["001", "002", "004", "005", "801", "802", "805", "903", "908"] llama_cpp_build = "631109b34" mlx_build = "a59cc23190f1e26fabcba8693607bb4c34b2ac2b" @@ -3780,7 +3784,9 @@ require_nonempty_thinking = true # 13, 13. # - main-601b8a65 (0.40.2-dynres-10-g601b8a6, the full build of main 601b8a65c, which v0.40.2-dynres.1 restamps): # PASS=11 SKIP=18 (runs/preflight-mlx-cuda-0402-main-g601b8a6.json). -version_pattern = '^0\.40\.2-dynres(\.\d+)?-\d+-g[0-9a-f]{7,40}$' +# Widened 2026-10-14 to 0.40.3 with cuda-0-40-2 (#488, Go-only): the v0.40.3 fold's head passed here on the +# production-like canary, PASS=11 SKIP=18, 0 cudaMemcpy errors with both GPUs visible. 0.40.4 stays refused. +version_pattern = '^0\.40\.[23]-dynres(\.\d+)?-\d+-g[0-9a-f]{7,40}$' mlx_build = "a59cc23190f1e26fabcba8693607bb4c34b2ac2b" platform = "mlx-cuda" reference_image = "maxusai/ollama:0.32.14-rc0-dynres-mlxfix" diff --git a/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-fold-canary-gpusall-g9f3baaf.json b/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-fold-canary-gpusall-g9f3baaf.json new file mode 100644 index 00000000000..48c526bd326 --- /dev/null +++ b/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-fold-canary-gpusall-g9f3baaf.json @@ -0,0 +1,472 @@ +{ + "meta": { + "host": "http://127.0.0.1:11532", + "platform": "cuda", + "image_tag": "maxusai/ollama:pr488-9f3baaf", + "started_utc": "20261013T141826", + "expectations_file": "expectations.toml", + "schema_version": 1, + "version": "0.40.2-dynres.1-6-g9f3baaf", + "profile": "cuda-0-40-2", + "patchset": [ + "001", + "002", + "004", + "005", + "801", + "802", + "805", + "903", + "908" + ], + "container": "ollama-ab-0402-canary488-gpusall", + "llama_cpp_build": "631109b34", + "mlx_build": "0.32.3-53-ga59cc23" + }, + "results": [ + { + "check": "version", + "arch": null, + "status": "PASS", + "summary": "0.40.2-dynres.1-6-g9f3baaf matches profile cuda-0-40-2", + "expected": "^0\\.40\\.2-dynres(\\.\\d+)?-\\d+-g[0-9a-f]{7,40}$", + "actual": "0.40.2-dynres.1-6-g9f3baaf" + }, + { + "check": "payload_pin", + "arch": null, + "status": "PASS", + "summary": "llama.cpp build 631109b34 matches the measured payload (docker exec)", + "expected": "631109b34", + "actual": "631109b34" + }, + { + "check": "toolchain_pin", + "arch": null, + "status": "SKIP", + "summary": "profile records no toolchain_build to assert against", + "diagnosis": "Add toolchain_build (e.g. \"rocm-7.2.4\" or \"cuda-13.0\") so a ROCm/CUDA bump under an unchanged payload fails loudly instead of inheriting these expectations. Platforms with no such toolchain (metal, cpu) leave it unset and keep this skip; the MLX library is pinned separately by mlx_build." + }, + { + "check": "image_tag", + "arch": null, + "status": "PASS", + "summary": "ollama-ab-0402-canary488-gpusall runs maxusai/ollama:pr488-9f3baaf", + "expected": "maxusai/ollama:pr488-9f3baaf", + "actual": "maxusai/ollama:pr488-9f3baaf" + }, + { + "check": "go_patch_marker", + "arch": null, + "status": "PASS", + "summary": "--image-max-tokens present in binary (1)", + "expected": 1, + "actual": 1 + }, + { + "check": "metal_tensor_host", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "metal_tensor_runtime", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "poison_probe", + "arch": null, + "status": "SKIP", + "summary": "no poison-probe expectation recorded for this profile" + }, + { + "check": "text_baseline", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "in-image text prefix = 19 (calibrated on 256x144+512x288); text-only = 20; text-only is +1 and MUST NOT be used (B8)", + "actual": 19 + }, + { + "check": "token_ladder", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 266, + "512x288": 266, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 3270 + }, + "actual": { + "256x144": 266, + "512x288": 266, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 3270 + }, + "rows": [ + { + "size": "256x144", + "expected": 266, + "actual": 266, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "512x288", + "expected": 266, + "actual": 266, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "1024x576", + "expected": 578, + "actual": 578, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 3270, + "actual": 3270, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "budget logged as custom on max/min (min 256*32^2=262144, max 3328*32^2=3407872)", + "expected": "min 256*32^2=262144, max 3328*32^2=3407872", + "actual": { + "min": "262144 (custom value)", + "max": "3407872 (custom value)" + } + }, + { + "check": "aspect_ladder", + "arch": "nemotron_h_omni", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "valid JSON after thinking (486 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 256, + "thinking_chars": 1623, + "eval_count": 486 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "pinned 3328 -> 3270 (ceiling 3328, +2 markers)", + "expected": 3270, + "actual": 3270, + "arms": [ + { + "arm": "pinned", + "pin": 3328, + "expected": 3270, + "actual": 3270, + "ok": true + }, + { + "arm": "ceiling_invariant", + "expected": "grid <= 3328", + "actual": 3268, + "ok": true + }, + { + "arm": "unpinned_control", + "expected": 2306, + "actual": 2306, + "ok": true + } + ] + }, + { + "check": "text_baseline", + "arch": "gemma4", + "status": "PASS", + "summary": "in-image text prefix = 19 (calibrated on 256x144+512x288); text-only = 19", + "actual": 19 + }, + { + "check": "token_ladder", + "arch": "gemma4", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "actual": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "rows": [ + { + "size": "256x144", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "512x288", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "1024x576", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "2048x1152", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "3072x1728", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "flat", + "expected_shape": "flat" + }, + { + "check": "payload_proof", + "arch": "gemma4", + "status": "PASS", + "summary": "budget logged as custom on max/min (min 70*48^2=161280, max 1120*48^2=2580480)", + "expected": "min 70*48^2=161280, max 1120*48^2=2580480", + "actual": { + "min": "161280 (custom value)", + "max": "2580480 (custom value)" + } + }, + { + "check": "aspect_ladder", + "arch": "gemma4", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "gemma4", + "status": "PASS", + "summary": "valid JSON after thinking (162 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 231, + "thinking_chars": 456, + "eval_count": 162 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "gemma4", + "status": "PASS", + "summary": "pinned 560 -> 529 (ceiling 1120, +2 markers)", + "expected": 529, + "actual": 529, + "arms": [ + { + "arm": "pinned", + "pin": 560, + "expected": 529, + "actual": 529, + "ok": true + }, + { + "arm": "ceiling_invariant", + "expected": "grid <= 1120", + "actual": 527, + "ok": true + } + ] + }, + { + "check": "text_baseline", + "arch": "qwen35", + "status": "PASS", + "summary": "in-image text prefix = 13 (calibrated on 256x144+512x288); text-only = 13", + "actual": 13 + }, + { + "check": "token_ladder", + "arch": "qwen35", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 1034, + "512x288": 1034, + "1024x576": 1034, + "2048x1152": 2306, + "3072x1728": 4082 + }, + "actual": { + "256x144": 1034, + "512x288": 1034, + "1024x576": 1034, + "2048x1152": 2306, + "3072x1728": 4082 + }, + "rows": [ + { + "size": "256x144", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 4082, + "actual": 4082, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "qwen35", + "status": "PASS", + "summary": "budget logged as custom on min (min 1024*32^2=1048576, max 4096*32^2=4194304)", + "expected": "min 1024*32^2=1048576, max 4096*32^2=4194304", + "actual": { + "min": "1048576 (custom value)", + "max": "4194304" + } + }, + { + "check": "aspect_ladder", + "arch": "qwen35", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "qwen35", + "status": "PASS", + "summary": "valid JSON after thinking (324 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 414, + "thinking_chars": 923, + "eval_count": 324 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "qwen35", + "status": "SKIP", + "summary": "pinned budget does not apply to this arch: image_min_tokens/image_max_tokens are arch-gated to gemma4 and nemotron_h_omni; qwen35's visionServerArgs branch ignores them." + }, + { + "check": "metal_tensor_payload", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "mlx_payload_pin", + "arch": null, + "status": "PASS", + "summary": "MLX build a59cc23 matches the measured payload (0.32.3-53-ga59cc23), from the shipped libmlx.so \u2014 no MLX request ran in this window, so binary/payload skew is not covered", + "expected": "a59cc23190f1e26fabcba8693607bb4c34b2ac2b", + "actual": "0.32.3-53-ga59cc23" + }, + { + "check": "endpoint_exclusive", + "arch": null, + "status": "PASS", + "summary": "no contention detected (worst queue wait 0.0s)", + "expected": "queue wait <= 10.0s", + "actual": "0.0s", + "models_loaded": [ + "qwen3.8:27b-q4_K_M" + ] + } + ], + "summary": { + "PASS": 20, + "SKIP": 9 + } +} \ No newline at end of file diff --git a/docs/maxusai/vision-suite/preflight/runs/preflight-mlx-cuda-0403-fold-canary-gpusall-g9f3baaf.json b/docs/maxusai/vision-suite/preflight/runs/preflight-mlx-cuda-0403-fold-canary-gpusall-g9f3baaf.json new file mode 100644 index 00000000000..de0e2c0193a --- /dev/null +++ b/docs/maxusai/vision-suite/preflight/runs/preflight-mlx-cuda-0403-fold-canary-gpusall-g9f3baaf.json @@ -0,0 +1,394 @@ +{ + "meta": { + "host": "http://127.0.0.1:11532", + "platform": "mlx-cuda", + "image_tag": "maxusai/ollama:pr488-9f3baaf", + "started_utc": "20261013T142206", + "expectations_file": "expectations.toml", + "schema_version": 1, + "version": "0.40.2-dynres.1-6-g9f3baaf", + "profile": "mlx-cuda-0-40-2", + "patchset": null, + "container": "ollama-ab-0402-canary488-gpusall", + "llama_cpp_build": "631109b34", + "mlx_build": "0.32.3-53-ga59cc23" + }, + "results": [ + { + "check": "version", + "arch": null, + "status": "PASS", + "summary": "0.40.2-dynres.1-6-g9f3baaf matches profile mlx-cuda-0-40-2", + "expected": "^0\\.40\\.2-dynres(\\.\\d+)?-\\d+-g[0-9a-f]{7,40}$", + "actual": "0.40.2-dynres.1-6-g9f3baaf" + }, + { + "check": "payload_pin", + "arch": null, + "status": "SKIP", + "summary": "profile records no llama_cpp_build to assert against", + "diagnosis": "Add llama_cpp_build to this profile so a llama.cpp bump fails loudly instead of inheriting stale expectations. See README.md." + }, + { + "check": "toolchain_pin", + "arch": null, + "status": "SKIP", + "summary": "profile records no toolchain_build to assert against", + "diagnosis": "Add toolchain_build (e.g. \"rocm-7.2.4\" or \"cuda-13.0\") so a ROCm/CUDA bump under an unchanged payload fails loudly instead of inheriting these expectations. Platforms with no such toolchain (metal, cpu) leave it unset and keep this skip; the MLX library is pinned separately by mlx_build." + }, + { + "check": "image_tag", + "arch": null, + "status": "PASS", + "summary": "ollama-ab-0402-canary488-gpusall runs maxusai/ollama:pr488-9f3baaf", + "expected": "maxusai/ollama:pr488-9f3baaf", + "actual": "maxusai/ollama:pr488-9f3baaf" + }, + { + "check": "go_patch_marker", + "arch": null, + "status": "PASS", + "summary": "--image-max-tokens present in binary (1)", + "expected": 1, + "actual": 1 + }, + { + "check": "metal_tensor_host", + "arch": null, + "status": "SKIP", + "summary": "does not apply on mlx-cuda" + }, + { + "check": "metal_tensor_runtime", + "arch": null, + "status": "SKIP", + "summary": "does not apply on mlx-cuda" + }, + { + "check": "poison_probe", + "arch": null, + "status": "SKIP", + "summary": "no poison-probe expectation recorded for this profile" + }, + { + "check": "text_baseline", + "arch": "gemma4", + "status": "PASS", + "summary": "in-image text prefix = 19 (calibrated on 256x144+512x288); text-only = 19", + "actual": 19 + }, + { + "check": "token_ladder", + "arch": "gemma4", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "actual": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "rows": [ + { + "size": "256x144", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "1024x576", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "flat", + "expected_shape": "flat" + }, + { + "check": "payload_proof", + "arch": "gemma4", + "status": "SKIP", + "summary": "budget/pixel values are not observable on this platform", + "diagnosis": "This block's budgets were established without a load log (the MLX runner emits none, on either backend) and are recorded as budgets_observed = false. Nothing here is waiting on a container; the block says how the values were established." + }, + { + "check": "aspect_ladder", + "arch": "gemma4", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "gemma4", + "status": "SKIP", + "summary": "no think+format expectation recorded" + }, + { + "check": "pinned_image_token_budget", + "arch": "gemma4", + "status": "SKIP", + "summary": "no pinned-budget expectation recorded for this arch", + "diagnosis": "Omitted deliberately: the 005 defect was found on nemotron and this arch's probe has not been measured. See README.md to add one." + }, + { + "check": "text_baseline", + "arch": "qwen35moe", + "status": "PASS", + "summary": "in-image text prefix = 13 (calibrated on 256x144+512x288); text-only = 13", + "actual": 13 + }, + { + "check": "token_ladder", + "arch": "qwen35moe", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "actual": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "rows": [ + { + "size": "256x144", + "expected": 68, + "actual": 68, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "512x288", + "expected": 146, + "actual": 146, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 578, + "actual": 578, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 5186, + "actual": 5186, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "budget/pixel values are not observable on this platform", + "diagnosis": "This block's budgets were established without a load log (the MLX runner emits none, on either backend) and are recorded as budgets_observed = false. Nothing here is waiting on a container; the block says how the values were established." + }, + { + "check": "aspect_ladder", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "no think+format expectation recorded" + }, + { + "check": "pinned_image_token_budget", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "no pinned-budget expectation recorded for this arch", + "diagnosis": "Omitted deliberately: the 005 defect was found on nemotron and this arch's probe has not been measured. See README.md to add one." + }, + { + "check": "text_baseline", + "arch": "qwen35", + "status": "PASS", + "summary": "in-image text prefix = 13 (calibrated on 256x144+512x288); text-only = 13", + "actual": 13 + }, + { + "check": "token_ladder", + "arch": "qwen35", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "actual": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "rows": [ + { + "size": "256x144", + "expected": 68, + "actual": 68, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 146, + "actual": 146, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 578, + "actual": 578, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 5186, + "actual": 5186, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "qwen35", + "status": "SKIP", + "summary": "budget/pixel values are not observable on this platform", + "diagnosis": "This block's budgets were established without a load log (the MLX runner emits none, on either backend) and are recorded as budgets_observed = false. Nothing here is waiting on a container; the block says how the values were established." + }, + { + "check": "aspect_ladder", + "arch": "qwen35", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "qwen35", + "status": "SKIP", + "summary": "no think+format expectation recorded" + }, + { + "check": "pinned_image_token_budget", + "arch": "qwen35", + "status": "SKIP", + "summary": "no pinned-budget expectation recorded for this arch", + "diagnosis": "Omitted deliberately: the 005 defect was found on nemotron and this arch's probe has not been measured. See README.md to add one." + }, + { + "check": "metal_tensor_payload", + "arch": null, + "status": "SKIP", + "summary": "does not apply on mlx-cuda" + }, + { + "check": "mlx_payload_pin", + "arch": null, + "status": "PASS", + "summary": "MLX build a59cc23 matches the measured payload (0.32.3-53-ga59cc23), from the runner's engine-init line", + "expected": "a59cc23190f1e26fabcba8693607bb4c34b2ac2b", + "actual": "0.32.3-53-ga59cc23" + }, + { + "check": "endpoint_exclusive", + "arch": null, + "status": "PASS", + "summary": "no contention detected (worst queue wait 0.0s)", + "expected": "queue wait <= 10.0s", + "actual": "0.0s", + "models_loaded": [ + "qwen3.8:27b-nvfp4" + ] + } + ], + "summary": { + "PASS": 11, + "SKIP": 18 + } +} \ No newline at end of file diff --git a/docs/maxusai/vision-suite/preflight/test_verdicts.py b/docs/maxusai/vision-suite/preflight/test_verdicts.py index d73055c7d1c..fdd6c23e25c 100644 --- a/docs/maxusai/vision-suite/preflight/test_verdicts.py +++ b/docs/maxusai/vision-suite/preflight/test_verdicts.py @@ -1744,11 +1744,15 @@ class TestCudaProfilesForTheV0402Fold(unittest.TestCase): "0.40.2-dynres-2-g19455c6", # the release image with #483's binary "0.40.2-dynres-10-g601b8a6", # main after the day's merges "0.40.2-dynres.1-0-gabcdef0", # a point tag ADR 0032 cuts for a release build + "0.40.2-dynres.1-4-g377f9d1", # production since 2026-10-13 23:39: the point tag + #487 + "0.40.2-dynres.1-6-g9f3baaf", # the v0.40.3 fold's head, measured here (#488) + "0.40.3-dynres-0-gabcdef0", # the v0.40.3 fold: Go-only, the same payload (ADR 0032) ) REJECT = ( "0.35.0-dynres-0-g043f441", # the previous payload: cuda-dynres-903 / mlx-cuda "0.40.2-dynres-0-g5623192-dirty", # a dirty tree describes nothing - "0.40.3-dynres-0-gabcdef0", # the next fold: measure first + "0.40.3-dynres-0-gabcdef0-dirty", # nor does a dirty v0.40.3 tree + "0.40.4-dynres-0-gabcdef0", # v0.40.4 moves llama.cpp to b11521: measure first ) @classmethod @@ -1771,7 +1775,8 @@ def test_the_pair_shares_one_pattern(self): def test_the_resolver_picks_these_profiles_and_keeps_0350_on_the_old_pair(self): import preflight for platform, new, old in (("cuda", "cuda-0-40-2", "cuda-dynres-903"), ("mlx-cuda", "mlx-cuda-0-40-2", "mlx-cuda")): - self.assertEqual(preflight.resolve_profile(self.exp, platform, "0.40.2-dynres-0-g5623192")[0], new) + for stamp in ("0.40.2-dynres-0-g5623192", "0.40.3-dynres-0-gabcdef0"): + self.assertEqual(preflight.resolve_profile(self.exp, platform, stamp)[0], new, stamp) self.assertEqual(preflight.resolve_profile(self.exp, platform, "0.35.0-dynres-0-g043f441")[0], old) def test_the_payload_pins(self):