diff --git a/README.md b/README.md index 2cecdcbfbc4..27e0b33dcf2 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ > | host | build | since | record | > |---|---|---|---| > | Apple Silicon, mlx-metal (`:11435`) | `0.40.2-dynres.1-0-g3960244`, both halves built at the point tag | 2026-10-13 22:01 | [#486](https://github.com/MaxusAI/ollama/pull/486) | -> | CUDA host, ai-server (`:11497`) | `0.40.2-dynres.1-4-g377f9d1`: the point tag plus [#487](https://github.com/MaxusAI/ollama/pull/487), a Go-only build of main `377f9d14` (image `main-377f9d14`) | 2026-10-13 23:39 | [#484](https://github.com/MaxusAI/ollama/pull/484), [#487](https://github.com/MaxusAI/ollama/pull/487) | +> | CUDA host, ai-server (`:11497`) | `0.40.2-dynres.1-17-g75c6d35`: the v0.40.3 fold ([#488](https://github.com/MaxusAI/ollama/pull/488)) on the point tag plus [#487](https://github.com/MaxusAI/ollama/pull/487), a Go-only build of main `75c6d350` (image `main-75c6d350`) | 2026-10-14 07:53 | [#487](https://github.com/MaxusAI/ollama/pull/487), [#488](https://github.com/MaxusAI/ollama/pull/488), [#492](https://github.com/MaxusAI/ollama/pull/492) | > | AMD gfx1151 | `v0.35.0-dynres`; the `v0.40.2-dynres.1` release build passes `rocm7-0-40-2`, and its deploy is scheduled after the [#477](https://github.com/MaxusAI/ollama/pull/477) campaign | 2026-10-01 | [#485](https://github.com/MaxusAI/ollama/pull/485) | > | H100 (CUDA) | `0.35.0-dynres-15-ga657392`, a build of `main` past the v0.35.0 tag | 2026-10-03 | [v0.35.0 record](docs/maxusai/tasks/upstream-sync-0.35.0.md) | > @@ -77,16 +77,18 @@ release_matrix.py --version 0.40.2-dynres.1 \ runs/preflight-mlx-metal-0402-dynres1-prod-g3960244.json \ runs/preflight-rocm7-0402-dynres1-release-g3960244.json \ - runs/preflight-cuda-0402-dynres1-487-prod-g377f9d1.json \ - runs/preflight-mlx-cuda-0402-dynres1-487-prod-g377f9d1.json + runs/preflight-cuda-0403-prod-g75c6d35.json \ + runs/preflight-mlx-cuda-0403-prod-g75c6d35.json The runs are listed by name because runs/ also holds #487's A/B runs and canaries, which are not release runs. - The CUDA host serves 0.40.2-dynres.1-4-g377f9d1 (main 377f9d14: the point tag plus #487, Go only); both of its - rows are production's own runs on that build, with both GPUs visible. --> + The CUDA host serves 0.40.2-dynres.1-17-g75c6d35 (main 75c6d350: the v0.40.3 fold on the point tag plus #487, Go + only); both of its rows are production's own runs on that build. The cuda row is the idle re-run + (preflight-cuda-0403-prod-g75c6d35.json); the first post-deploy run, -contended.json, failed gemma4's ladder and + pinned budget by a constant +573 while a client's gemma4 q8 shared the GPU and capped image_max_tokens to n_ubatch. --> | surface | Build identity | MLX build pin | Toolchain pin | M5 tensor path | Image size ladder | Aspect ladder | Pinned image budget | thinking on/off | Output quality | fp16 overflow canary | Runner isolation | measured on | |---|---|---|---|---|---|---|---|---|---|---|---|---| -| **cuda** | green | green | skipped | n/a | green | skipped | green | green | not run | skipped | green | `0.40.2-dynres.1-4-g377f9d1` | -| **mlx-cuda** | skipped | green | skipped | n/a | green | skipped | skipped | skipped | not run | skipped | green | `0.40.2-dynres.1-4-g377f9d1` | +| **cuda** | green | green | skipped | n/a | green | skipped | green | green | not run | skipped | green | `0.40.2-dynres.1-17-g75c6d35` | +| **mlx-cuda** | skipped | green | skipped | n/a | green | skipped | skipped | skipped | not run | skipped | green | `0.40.2-dynres.1-17-g75c6d35` | | **mlx-metal** | skipped | green | skipped | green | green | green | skipped | green | skipped | skipped | green | `0.40.2-dynres.1-0-g3960244` | | **apple-silicon-mlx** | not run | not run | not run | not run | not run | not run | not run | not run | not run | not run | not run | — | | **rocm7** | green | skipped | green | n/a | green | green | green | green | green | green | green | `0.40.2-dynres.1-0-g3960244` | diff --git a/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-canary-gpusall-g75c6d35.json b/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-canary-gpusall-g75c6d35.json new file mode 100644 index 00000000000..7a6347a9c21 --- /dev/null +++ b/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-canary-gpusall-g75c6d35.json @@ -0,0 +1,472 @@ +{ + "meta": { + "host": "http://127.0.0.1:11532", + "platform": "cuda", + "image_tag": "maxusai/ollama:main-75c6d350", + "started_utc": "20261013T214231", + "expectations_file": "expectations.toml", + "schema_version": 1, + "version": "0.40.2-dynres.1-17-g75c6d35", + "profile": "cuda-0-40-2", + "patchset": [ + "001", + "002", + "004", + "005", + "801", + "802", + "805", + "903", + "908" + ], + "container": "ollama-ab-0402-canary0403-gpusall", + "llama_cpp_build": "631109b34", + "mlx_build": "0.32.3-53-ga59cc23" + }, + "results": [ + { + "check": "version", + "arch": null, + "status": "PASS", + "summary": "0.40.2-dynres.1-17-g75c6d35 matches profile cuda-0-40-2", + "expected": "^0\\.40\\.2-dynres(\\.\\d+)?-\\d+-g[0-9a-f]{7,40}$", + "actual": "0.40.2-dynres.1-17-g75c6d35" + }, + { + "check": "payload_pin", + "arch": null, + "status": "PASS", + "summary": "llama.cpp build 631109b34 matches the measured payload (docker exec)", + "expected": "631109b34", + "actual": "631109b34" + }, + { + "check": "toolchain_pin", + "arch": null, + "status": "SKIP", + "summary": "profile records no toolchain_build to assert against", + "diagnosis": "Add toolchain_build (e.g. \"rocm-7.2.4\" or \"cuda-13.0\") so a ROCm/CUDA bump under an unchanged payload fails loudly instead of inheriting these expectations. Platforms with no such toolchain (metal, cpu) leave it unset and keep this skip; the MLX library is pinned separately by mlx_build." + }, + { + "check": "image_tag", + "arch": null, + "status": "PASS", + "summary": "ollama-ab-0402-canary0403-gpusall runs maxusai/ollama:main-75c6d350", + "expected": "maxusai/ollama:main-75c6d350", + "actual": "maxusai/ollama:main-75c6d350" + }, + { + "check": "go_patch_marker", + "arch": null, + "status": "PASS", + "summary": "--image-max-tokens present in binary (1)", + "expected": 1, + "actual": 1 + }, + { + "check": "metal_tensor_host", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "metal_tensor_runtime", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "poison_probe", + "arch": null, + "status": "SKIP", + "summary": "no poison-probe expectation recorded for this profile" + }, + { + "check": "text_baseline", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "in-image text prefix = 19 (calibrated on 256x144+512x288); text-only = 20; text-only is +1 and MUST NOT be used (B8)", + "actual": 19 + }, + { + "check": "token_ladder", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 266, + "512x288": 266, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 3270 + }, + "actual": { + "256x144": 266, + "512x288": 266, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 3270 + }, + "rows": [ + { + "size": "256x144", + "expected": 266, + "actual": 266, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "512x288", + "expected": 266, + "actual": 266, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 578, + "actual": 578, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 3270, + "actual": 3270, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "budget logged as custom on max/min (min 256*32^2=262144, max 3328*32^2=3407872)", + "expected": "min 256*32^2=262144, max 3328*32^2=3407872", + "actual": { + "min": "262144 (custom value)", + "max": "3407872 (custom value)" + } + }, + { + "check": "aspect_ladder", + "arch": "nemotron_h_omni", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "valid JSON after thinking (486 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 256, + "thinking_chars": 1623, + "eval_count": 486 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "pinned 3328 -> 3270 (ceiling 3328, +2 markers)", + "expected": 3270, + "actual": 3270, + "arms": [ + { + "arm": "pinned", + "pin": 3328, + "expected": 3270, + "actual": 3270, + "ok": true + }, + { + "arm": "ceiling_invariant", + "expected": "grid <= 3328", + "actual": 3268, + "ok": true + }, + { + "arm": "unpinned_control", + "expected": 2306, + "actual": 2306, + "ok": true + } + ] + }, + { + "check": "text_baseline", + "arch": "gemma4", + "status": "PASS", + "summary": "in-image text prefix = 19 (calibrated on 256x144+512x288); text-only = 19", + "actual": 19 + }, + { + "check": "token_ladder", + "arch": "gemma4", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "actual": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "rows": [ + { + "size": "256x144", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "flat", + "expected_shape": "flat" + }, + { + "check": "payload_proof", + "arch": "gemma4", + "status": "PASS", + "summary": "budget logged as custom on max/min (min 70*48^2=161280, max 1120*48^2=2580480)", + "expected": "min 70*48^2=161280, max 1120*48^2=2580480", + "actual": { + "min": "161280 (custom value)", + "max": "2580480 (custom value)" + } + }, + { + "check": "aspect_ladder", + "arch": "gemma4", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "gemma4", + "status": "PASS", + "summary": "valid JSON after thinking (162 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 231, + "thinking_chars": 456, + "eval_count": 162 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "gemma4", + "status": "PASS", + "summary": "pinned 560 -> 529 (ceiling 1120, +2 markers)", + "expected": 529, + "actual": 529, + "arms": [ + { + "arm": "pinned", + "pin": 560, + "expected": 529, + "actual": 529, + "ok": true + }, + { + "arm": "ceiling_invariant", + "expected": "grid <= 1120", + "actual": 527, + "ok": true + } + ] + }, + { + "check": "text_baseline", + "arch": "qwen35", + "status": "PASS", + "summary": "in-image text prefix = 13 (calibrated on 256x144+512x288); text-only = 13", + "actual": 13 + }, + { + "check": "token_ladder", + "arch": "qwen35", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 1034, + "512x288": 1034, + "1024x576": 1034, + "2048x1152": 2306, + "3072x1728": 4082 + }, + "actual": { + "256x144": 1034, + "512x288": 1034, + "1024x576": 1034, + "2048x1152": 2306, + "3072x1728": 4082 + }, + "rows": [ + { + "size": "256x144", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "1024x576", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 4082, + "actual": 4082, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "qwen35", + "status": "PASS", + "summary": "budget logged as custom on min (min 1024*32^2=1048576, max 4096*32^2=4194304)", + "expected": "min 1024*32^2=1048576, max 4096*32^2=4194304", + "actual": { + "min": "1048576 (custom value)", + "max": "4194304" + } + }, + { + "check": "aspect_ladder", + "arch": "qwen35", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "qwen35", + "status": "PASS", + "summary": "valid JSON after thinking (324 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 414, + "thinking_chars": 923, + "eval_count": 324 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "qwen35", + "status": "SKIP", + "summary": "pinned budget does not apply to this arch: image_min_tokens/image_max_tokens are arch-gated to gemma4 and nemotron_h_omni; qwen35's visionServerArgs branch ignores them." + }, + { + "check": "metal_tensor_payload", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "mlx_payload_pin", + "arch": null, + "status": "PASS", + "summary": "MLX build a59cc23 matches the measured payload (0.32.3-53-ga59cc23), from the shipped libmlx.so \u2014 no MLX request ran in this window, so binary/payload skew is not covered", + "expected": "a59cc23190f1e26fabcba8693607bb4c34b2ac2b", + "actual": "0.32.3-53-ga59cc23" + }, + { + "check": "endpoint_exclusive", + "arch": null, + "status": "PASS", + "summary": "no contention detected (worst queue wait 0.0s)", + "expected": "queue wait <= 10.0s", + "actual": "0.0s", + "models_loaded": [ + "qwen3.8:27b-q4_K_M" + ] + } + ], + "summary": { + "PASS": 20, + "SKIP": 9 + } +} \ No newline at end of file diff --git a/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-prod-g75c6d35-contended.json b/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-prod-g75c6d35-contended.json new file mode 100644 index 00000000000..7856dec93a1 --- /dev/null +++ b/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-prod-g75c6d35-contended.json @@ -0,0 +1,476 @@ +{ + "meta": { + "host": "http://127.0.0.1:11497", + "platform": "cuda", + "image_tag": "maxusai/ollama:main-75c6d350", + "started_utc": "20261013T215652", + "expectations_file": "expectations.toml", + "schema_version": 1, + "version": "0.40.2-dynres.1-17-g75c6d35", + "profile": "cuda-0-40-2", + "patchset": [ + "001", + "002", + "004", + "005", + "801", + "802", + "805", + "903", + "908" + ], + "container": "ollama-0.40.2-dynres.1-17-g75c6d35", + "llama_cpp_build": "631109b34", + "mlx_build": "0.32.3-53-ga59cc23" + }, + "results": [ + { + "check": "version", + "arch": null, + "status": "PASS", + "summary": "0.40.2-dynres.1-17-g75c6d35 matches profile cuda-0-40-2", + "expected": "^0\\.40\\.2-dynres(\\.\\d+)?-\\d+-g[0-9a-f]{7,40}$", + "actual": "0.40.2-dynres.1-17-g75c6d35" + }, + { + "check": "payload_pin", + "arch": null, + "status": "PASS", + "summary": "llama.cpp build 631109b34 matches the measured payload (docker exec)", + "expected": "631109b34", + "actual": "631109b34" + }, + { + "check": "toolchain_pin", + "arch": null, + "status": "SKIP", + "summary": "profile records no toolchain_build to assert against", + "diagnosis": "Add toolchain_build (e.g. \"rocm-7.2.4\" or \"cuda-13.0\") so a ROCm/CUDA bump under an unchanged payload fails loudly instead of inheriting these expectations. Platforms with no such toolchain (metal, cpu) leave it unset and keep this skip; the MLX library is pinned separately by mlx_build." + }, + { + "check": "image_tag", + "arch": null, + "status": "PASS", + "summary": "ollama-0.40.2-dynres.1-17-g75c6d35 runs maxusai/ollama:main-75c6d350", + "expected": "maxusai/ollama:main-75c6d350", + "actual": "maxusai/ollama:main-75c6d350" + }, + { + "check": "go_patch_marker", + "arch": null, + "status": "PASS", + "summary": "--image-max-tokens present in binary (1)", + "expected": 1, + "actual": 1 + }, + { + "check": "metal_tensor_host", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "metal_tensor_runtime", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "poison_probe", + "arch": null, + "status": "SKIP", + "summary": "no poison-probe expectation recorded for this profile" + }, + { + "check": "text_baseline", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "in-image text prefix = 19 (calibrated on 256x144+512x288); text-only = 20; text-only is +1 and MUST NOT be used (B8)", + "actual": 19 + }, + { + "check": "token_ladder", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 266, + "512x288": 266, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 3270 + }, + "actual": { + "256x144": 266, + "512x288": 266, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 3270 + }, + "rows": [ + { + "size": "256x144", + "expected": 266, + "actual": 266, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "512x288", + "expected": 266, + "actual": 266, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 578, + "actual": 578, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 3270, + "actual": 3270, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "budget logged as custom on max/min (min 256*32^2=262144, max 3328*32^2=3407872)", + "expected": "min 256*32^2=262144, max 3328*32^2=3407872", + "actual": { + "min": "262144 (custom value)", + "max": "3407872 (custom value)" + } + }, + { + "check": "aspect_ladder", + "arch": "nemotron_h_omni", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "valid JSON after thinking (486 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 256, + "thinking_chars": 1623, + "eval_count": 486 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "pinned 3328 -> 3270 (ceiling 3328, +2 markers)", + "expected": 3270, + "actual": 3270, + "arms": [ + { + "arm": "pinned", + "pin": 3328, + "expected": 3270, + "actual": 3270, + "ok": true + }, + { + "arm": "ceiling_invariant", + "expected": "grid <= 3328", + "actual": 3268, + "ok": true + }, + { + "arm": "unpinned_control", + "expected": 2306, + "actual": 2306, + "ok": true + } + ] + }, + { + "check": "text_baseline", + "arch": "gemma4", + "status": "PASS", + "summary": "in-image text prefix = -554 (calibrated on 256x144+512x288); text-only = 19; text-only is +573 and MUST NOT be used (B8)", + "actual": -554 + }, + { + "check": "token_ladder", + "arch": "gemma4", + "status": "FAIL", + "summary": "0/5 geometries within +/-2 (shape: flat, expected flat)", + "expected": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "actual": { + "256x144": 1675, + "512x288": 1675, + "1024x576": 1675, + "2048x1152": 1675, + "3072x1728": 1675 + }, + "diagnosis": "Values moved but the shape is right: most likely a budget or preprocessing change. Re-measure and update expectations.toml if the new numbers are intended.", + "rows": [ + { + "size": "256x144", + "expected": 1102, + "actual": 1675, + "delta": 573, + "ok": false, + "queue_wait_s": 0.01 + }, + { + "size": "512x288", + "expected": 1102, + "actual": 1675, + "delta": 573, + "ok": false, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 1102, + "actual": 1675, + "delta": 573, + "ok": false, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 1102, + "actual": 1675, + "delta": 573, + "ok": false, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 1102, + "actual": 1675, + "delta": 573, + "ok": false, + "queue_wait_s": 0.01 + } + ], + "shape": "flat", + "expected_shape": "flat" + }, + { + "check": "payload_proof", + "arch": "gemma4", + "status": "PASS", + "summary": "budget logged as custom on max/min (min 70*48^2=161280, max 1120*48^2=2580480)", + "expected": "min 70*48^2=161280, max 1120*48^2=2580480", + "actual": { + "min": "161280 (custom value)", + "max": "2580480 (custom value)" + } + }, + { + "check": "aspect_ladder", + "arch": "gemma4", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "gemma4", + "status": "PASS", + "summary": "valid JSON after thinking (162 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 231, + "thinking_chars": 456, + "eval_count": 162 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "gemma4", + "status": "FAIL", + "summary": "pinned 560 -> 1102 (ceiling 1120, +2 markers)", + "expected": 529, + "actual": 1102, + "diagnosis": "Pinned cost moved (529 -> 1102) but stays under the ceiling, so this is a behaviour change, not the 005 defect. Re-measure and update expectations.toml if intended.", + "arms": [ + { + "arm": "pinned", + "pin": 560, + "expected": 529, + "actual": 1102, + "ok": false + }, + { + "arm": "ceiling_invariant", + "expected": "grid <= 1120", + "actual": 1100, + "ok": true + } + ] + }, + { + "check": "text_baseline", + "arch": "qwen35", + "status": "PASS", + "summary": "in-image text prefix = 13 (calibrated on 256x144+512x288); text-only = 13", + "actual": 13 + }, + { + "check": "token_ladder", + "arch": "qwen35", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 1034, + "512x288": 1034, + "1024x576": 1034, + "2048x1152": 2306, + "3072x1728": 4082 + }, + "actual": { + "256x144": 1034, + "512x288": 1034, + "1024x576": 1034, + "2048x1152": 2306, + "3072x1728": 4082 + }, + "rows": [ + { + "size": "256x144", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 4082, + "actual": 4082, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "qwen35", + "status": "PASS", + "summary": "budget logged as custom on min (min 1024*32^2=1048576, max 4096*32^2=4194304)", + "expected": "min 1024*32^2=1048576, max 4096*32^2=4194304", + "actual": { + "min": "1048576 (custom value)", + "max": "4194304" + } + }, + { + "check": "aspect_ladder", + "arch": "qwen35", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "qwen35", + "status": "PASS", + "summary": "valid JSON after thinking (324 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 414, + "thinking_chars": 923, + "eval_count": 324 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "qwen35", + "status": "SKIP", + "summary": "pinned budget does not apply to this arch: image_min_tokens/image_max_tokens are arch-gated to gemma4 and nemotron_h_omni; qwen35's visionServerArgs branch ignores them." + }, + { + "check": "metal_tensor_payload", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "mlx_payload_pin", + "arch": null, + "status": "PASS", + "summary": "MLX build a59cc23 matches the measured payload (0.32.3-53-ga59cc23), from the shipped libmlx.so \u2014 no MLX request ran in this window, so binary/payload skew is not covered", + "expected": "a59cc23190f1e26fabcba8693607bb4c34b2ac2b", + "actual": "0.32.3-53-ga59cc23" + }, + { + "check": "endpoint_exclusive", + "arch": null, + "status": "PASS", + "summary": "no contention detected (worst queue wait 0.0s)", + "expected": "queue wait <= 10.0s", + "actual": "0.0s", + "models_loaded": [ + "qwen3.8:27b-q4_K_M", + "gemma4:31b-it-q4_K_M" + ] + } + ], + "summary": { + "PASS": 18, + "SKIP": 9, + "FAIL": 2 + } +} \ No newline at end of file diff --git a/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-prod-g75c6d35.host-profile.json b/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-prod-g75c6d35.host-profile.json new file mode 100644 index 00000000000..8d37701bc59 --- /dev/null +++ b/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-prod-g75c6d35.host-profile.json @@ -0,0 +1,145 @@ +{ + "schema": "host-profile/1", + "collected_at": "2026-10-13T21:56:52Z", + "label": "ai-server/mlx-cuda", + "label_host": "ai-server", + "label_surface": "mlx-cuda", + "machine": { + "vendor": "ASUSTeK COMPUTER INC.", + "model": "WS-C621E-SAGE Series", + "model_id": null, + "virtualized": false, + "hypervisor": null + }, + "cloud": null, + "os": { + "family": "linux", + "name": "Ubuntu", + "version": "24.04", + "pretty_name": "Ubuntu 24.04.5 LTS", + "kernel": "6.8.0-134-generic", + "arch": "x86_64" + }, + "cpu": { + "vendor": "intel", + "model": "Intel(R) Xeon(R) Silver 4110 CPU @ 2.10GHz", + "logical_cpus": 32, + "sockets": 2, + "cores": 16, + "threads_per_core": 2, + "performance_cores": null, + "efficiency_cores": null, + "numa_nodes": 2, + "l3_cache_mib": 22.0, + "max_mhz": 3000.0, + "flags": { + "avx2": true, + "avx512f": true, + "avx512_vnni": false, + "avx512_bf16": false, + "avx512_fp16": false, + "amx_tile": false, + "amx_bf16": false, + "amx_int8": false + } + }, + "memory_total_gib": 754.5, + "gpu_count": 2, + "gpus": [ + { + "vendor": "nvidia", + "name": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", + "pci_name": "Device 2bb1", + "pci_bus_id": "0000:3b:00.0", + "pci_vendor_id": "0x10de", + "pci_device_id": "0x2bb1", + "driver": "nvidia", + "memory_total_mib": 97887, + "memory_gtt_mib": null, + "unified_memory": null, + "compute_capability": "12.0", + "gfx_target": null, + "gpu_cores": null, + "metal_family": null, + "vbios": "98.02.81.00.07", + "power_limit_w": 600.0, + "power_max_limit_w": 600.0, + "max_core_clock_mhz": 3090, + "max_memory_clock_mhz": 14001, + "ecc_enabled": false, + "mig_enabled": null, + "persistence_enabled": false, + "pcie_link_gen": 3, + "pcie_link_width": 16, + "pcie_device_max_gen": 5, + "pcie_device_max_width": 16, + "pcie_upstream_max_gen": 3, + "pcie_upstream_max_width": 16, + "pcie_gen_now": 1, + "pcie_width_now": 16, + "numa_node": 0, + "local_cpus": "0-7,16-23", + "index": 0 + }, + { + "vendor": "nvidia", + "name": "NVIDIA GeForce RTX 2080 Ti", + "pci_name": "TU102 [GeForce RTX 2080 Ti]", + "pci_bus_id": "0000:86:00.0", + "pci_vendor_id": "0x10de", + "pci_device_id": "0x1e04", + "driver": "nvidia", + "memory_total_mib": 11264, + "memory_gtt_mib": null, + "unified_memory": null, + "compute_capability": "7.5", + "gfx_target": null, + "gpu_cores": null, + "metal_family": null, + "vbios": "90.02.17.00.B2", + "power_limit_w": 250.0, + "power_max_limit_w": 280.0, + "max_core_clock_mhz": 2100, + "max_memory_clock_mhz": 7000, + "ecc_enabled": null, + "mig_enabled": null, + "persistence_enabled": false, + "pcie_link_gen": 3, + "pcie_link_width": 16, + "pcie_device_max_gen": 3, + "pcie_device_max_width": 16, + "pcie_upstream_max_gen": 3, + "pcie_upstream_max_width": 16, + "pcie_gen_now": 1, + "pcie_width_now": 16, + "numa_node": 1, + "local_cpus": "8-15,24-31", + "index": 1 + } + ], + "filesystems": [ + { + "role": "models", + "fstype": "btrfs", + "size_gib": 7452.0, + "raid_level": null, + "raid_devices": null + } + ], + "software": { + "nvidia_driver_version": "580.126.18", + "nvidia_kernel_module": "open", + "cuda_driver_version": "13.0", + "cuda_toolkit_version": "12.1.105", + "cudnn_version": "9.13.1.26", + "nccl_version": "2.28.3", + "rocm_version": null, + "amdgpu_driver_version": null, + "docker_version": "28.4.0", + "nvidia_container_toolkit_version": "1.20.1", + "dcgm_version": "3.3.9", + "ollama_version": "0.40.2-dynres.1-17-g75c6d35", + "go_version": null, + "python_version": "3.12.3" + } +} diff --git a/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-prod-g75c6d35.json b/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-prod-g75c6d35.json new file mode 100644 index 00000000000..7528e4dac4c --- /dev/null +++ b/docs/maxusai/vision-suite/preflight/runs/preflight-cuda-0403-prod-g75c6d35.json @@ -0,0 +1,474 @@ +{ + "meta": { + "host": "http://127.0.0.1:11497", + "platform": "cuda", + "image_tag": "maxusai/ollama:main-75c6d350", + "started_utc": "20261013T224619", + "expectations_file": "expectations.toml", + "schema_version": 1, + "version": "0.40.2-dynres.1-17-g75c6d35", + "profile": "cuda-0-40-2", + "patchset": [ + "001", + "002", + "004", + "005", + "801", + "802", + "805", + "903", + "908" + ], + "container": "ollama-0.40.2-dynres.1-17-g75c6d35", + "llama_cpp_build": "631109b34", + "mlx_build": "0.32.3-53-ga59cc23" + }, + "results": [ + { + "check": "version", + "arch": null, + "status": "PASS", + "summary": "0.40.2-dynres.1-17-g75c6d35 matches profile cuda-0-40-2", + "expected": "^0\\.40\\.2-dynres(\\.\\d+)?-\\d+-g[0-9a-f]{7,40}$", + "actual": "0.40.2-dynres.1-17-g75c6d35" + }, + { + "check": "payload_pin", + "arch": null, + "status": "PASS", + "summary": "llama.cpp build 631109b34 matches the measured payload (docker exec)", + "expected": "631109b34", + "actual": "631109b34" + }, + { + "check": "toolchain_pin", + "arch": null, + "status": "SKIP", + "summary": "profile records no toolchain_build to assert against", + "diagnosis": "Add toolchain_build (e.g. \"rocm-7.2.4\" or \"cuda-13.0\") so a ROCm/CUDA bump under an unchanged payload fails loudly instead of inheriting these expectations. Platforms with no such toolchain (metal, cpu) leave it unset and keep this skip; the MLX library is pinned separately by mlx_build." + }, + { + "check": "image_tag", + "arch": null, + "status": "PASS", + "summary": "ollama-0.40.2-dynres.1-17-g75c6d35 runs maxusai/ollama:main-75c6d350", + "expected": "maxusai/ollama:main-75c6d350", + "actual": "maxusai/ollama:main-75c6d350" + }, + { + "check": "go_patch_marker", + "arch": null, + "status": "PASS", + "summary": "--image-max-tokens present in binary (1)", + "expected": 1, + "actual": 1 + }, + { + "check": "metal_tensor_host", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "metal_tensor_runtime", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "poison_probe", + "arch": null, + "status": "SKIP", + "summary": "no poison-probe expectation recorded for this profile" + }, + { + "check": "text_baseline", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "in-image text prefix = 19 (calibrated on 256x144+512x288); text-only = 20; text-only is +1 and MUST NOT be used (B8)", + "actual": 19 + }, + { + "check": "token_ladder", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 266, + "512x288": 266, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 3270 + }, + "actual": { + "256x144": 266, + "512x288": 266, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 3270 + }, + "rows": [ + { + "size": "256x144", + "expected": 266, + "actual": 266, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 266, + "actual": 266, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "1024x576", + "expected": 578, + "actual": 578, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 3270, + "actual": 3270, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "budget logged as custom on max/min (min 256*32^2=262144, max 3328*32^2=3407872)", + "expected": "min 256*32^2=262144, max 3328*32^2=3407872", + "actual": { + "min": "262144 (custom value)", + "max": "3407872 (custom value)" + } + }, + { + "check": "aspect_ladder", + "arch": "nemotron_h_omni", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "valid JSON after thinking (486 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 256, + "thinking_chars": 1623, + "eval_count": 486 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "nemotron_h_omni", + "status": "PASS", + "summary": "pinned 3328 -> 3270 (ceiling 3328, +2 markers)", + "expected": 3270, + "actual": 3270, + "arms": [ + { + "arm": "pinned", + "pin": 3328, + "expected": 3270, + "actual": 3270, + "ok": true + }, + { + "arm": "ceiling_invariant", + "expected": "grid <= 3328", + "actual": 3268, + "ok": true + }, + { + "arm": "unpinned_control", + "expected": 2306, + "actual": 2306, + "ok": true + } + ] + }, + { + "check": "text_baseline", + "arch": "gemma4", + "status": "PASS", + "summary": "in-image text prefix = 19 (calibrated on 256x144+512x288); text-only = 19", + "actual": 19 + }, + { + "check": "token_ladder", + "arch": "gemma4", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "actual": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "rows": [ + { + "size": "256x144", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "1024x576", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "flat", + "expected_shape": "flat" + }, + { + "check": "payload_proof", + "arch": "gemma4", + "status": "PASS", + "summary": "budget logged as custom on max/min (min 70*48^2=161280, max 1120*48^2=2580480)", + "expected": "min 70*48^2=161280, max 1120*48^2=2580480", + "actual": { + "min": "161280 (custom value)", + "max": "2580480 (custom value)" + } + }, + { + "check": "aspect_ladder", + "arch": "gemma4", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "gemma4", + "status": "PASS", + "summary": "valid JSON after thinking (162 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 231, + "thinking_chars": 456, + "eval_count": 162 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "gemma4", + "status": "PASS", + "summary": "pinned 560 -> 529 (ceiling 1120, +2 markers)", + "expected": 529, + "actual": 529, + "arms": [ + { + "arm": "pinned", + "pin": 560, + "expected": 529, + "actual": 529, + "ok": true + }, + { + "arm": "ceiling_invariant", + "expected": "grid <= 1120", + "actual": 527, + "ok": true + } + ] + }, + { + "check": "text_baseline", + "arch": "qwen35", + "status": "PASS", + "summary": "in-image text prefix = 13 (calibrated on 256x144+512x288); text-only = 13", + "actual": 13 + }, + { + "check": "token_ladder", + "arch": "qwen35", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 1034, + "512x288": 1034, + "1024x576": 1034, + "2048x1152": 2306, + "3072x1728": 4082 + }, + "actual": { + "256x144": 1034, + "512x288": 1034, + "1024x576": 1034, + "2048x1152": 2306, + "3072x1728": 4082 + }, + "rows": [ + { + "size": "256x144", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "512x288", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 1034, + "actual": 1034, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 4082, + "actual": 4082, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "qwen35", + "status": "PASS", + "summary": "budget logged as custom on min (min 1024*32^2=1048576, max 4096*32^2=4194304)", + "expected": "min 1024*32^2=1048576, max 4096*32^2=4194304", + "actual": { + "min": "1048576 (custom value)", + "max": "4194304" + } + }, + { + "check": "aspect_ladder", + "arch": "qwen35", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "qwen35", + "status": "PASS", + "summary": "valid JSON after thinking (324 tokens)", + "expected": "non-empty JSON response with think:true + format:json", + "actual": { + "response_chars": 414, + "thinking_chars": 923, + "eval_count": 324 + } + }, + { + "check": "pinned_image_token_budget", + "arch": "qwen35", + "status": "SKIP", + "summary": "pinned budget does not apply to this arch: image_min_tokens/image_max_tokens are arch-gated to gemma4 and nemotron_h_omni; qwen35's visionServerArgs branch ignores them." + }, + { + "check": "metal_tensor_payload", + "arch": null, + "status": "SKIP", + "summary": "does not apply on cuda" + }, + { + "check": "mlx_payload_pin", + "arch": null, + "status": "PASS", + "summary": "MLX build a59cc23 matches the measured payload (0.32.3-53-ga59cc23), from the shipped libmlx.so \u2014 no MLX request ran in this window, so binary/payload skew is not covered", + "expected": "a59cc23190f1e26fabcba8693607bb4c34b2ac2b", + "actual": "0.32.3-53-ga59cc23" + }, + { + "check": "endpoint_exclusive", + "arch": null, + "status": "PASS", + "summary": "no contention detected (worst queue wait 0.0s)", + "expected": "queue wait <= 10.0s", + "actual": "0.0s", + "models_loaded": [ + "qwen3.8:27b-q4_K_M", + "gemma4:31b-it-q4_K_M", + "nemotron3:33b-q8" + ] + } + ], + "summary": { + "PASS": 20, + "SKIP": 9 + } +} \ No newline at end of file diff --git a/docs/maxusai/vision-suite/preflight/runs/preflight-mlx-cuda-0403-canary-gpusall-g75c6d35.json b/docs/maxusai/vision-suite/preflight/runs/preflight-mlx-cuda-0403-canary-gpusall-g75c6d35.json new file mode 100644 index 00000000000..7ca90811f3b --- /dev/null +++ b/docs/maxusai/vision-suite/preflight/runs/preflight-mlx-cuda-0403-canary-gpusall-g75c6d35.json @@ -0,0 +1,394 @@ +{ + "meta": { + "host": "http://127.0.0.1:11532", + "platform": "mlx-cuda", + "image_tag": "maxusai/ollama:main-75c6d350", + "started_utc": "20261013T214618", + "expectations_file": "expectations.toml", + "schema_version": 1, + "version": "0.40.2-dynres.1-17-g75c6d35", + "profile": "mlx-cuda-0-40-2", + "patchset": null, + "container": "ollama-ab-0402-canary0403-gpusall", + "llama_cpp_build": "631109b34", + "mlx_build": "0.32.3-53-ga59cc23" + }, + "results": [ + { + "check": "version", + "arch": null, + "status": "PASS", + "summary": "0.40.2-dynres.1-17-g75c6d35 matches profile mlx-cuda-0-40-2", + "expected": "^0\\.40\\.2-dynres(\\.\\d+)?-\\d+-g[0-9a-f]{7,40}$", + "actual": "0.40.2-dynres.1-17-g75c6d35" + }, + { + "check": "payload_pin", + "arch": null, + "status": "SKIP", + "summary": "profile records no llama_cpp_build to assert against", + "diagnosis": "Add llama_cpp_build to this profile so a llama.cpp bump fails loudly instead of inheriting stale expectations. See README.md." + }, + { + "check": "toolchain_pin", + "arch": null, + "status": "SKIP", + "summary": "profile records no toolchain_build to assert against", + "diagnosis": "Add toolchain_build (e.g. \"rocm-7.2.4\" or \"cuda-13.0\") so a ROCm/CUDA bump under an unchanged payload fails loudly instead of inheriting these expectations. Platforms with no such toolchain (metal, cpu) leave it unset and keep this skip; the MLX library is pinned separately by mlx_build." + }, + { + "check": "image_tag", + "arch": null, + "status": "PASS", + "summary": "ollama-ab-0402-canary0403-gpusall runs maxusai/ollama:main-75c6d350", + "expected": "maxusai/ollama:main-75c6d350", + "actual": "maxusai/ollama:main-75c6d350" + }, + { + "check": "go_patch_marker", + "arch": null, + "status": "PASS", + "summary": "--image-max-tokens present in binary (1)", + "expected": 1, + "actual": 1 + }, + { + "check": "metal_tensor_host", + "arch": null, + "status": "SKIP", + "summary": "does not apply on mlx-cuda" + }, + { + "check": "metal_tensor_runtime", + "arch": null, + "status": "SKIP", + "summary": "does not apply on mlx-cuda" + }, + { + "check": "poison_probe", + "arch": null, + "status": "SKIP", + "summary": "no poison-probe expectation recorded for this profile" + }, + { + "check": "text_baseline", + "arch": "gemma4", + "status": "PASS", + "summary": "in-image text prefix = 19 (calibrated on 256x144+512x288); text-only = 19", + "actual": 19 + }, + { + "check": "token_ladder", + "arch": "gemma4", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "actual": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "rows": [ + { + "size": "256x144", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "512x288", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "flat", + "expected_shape": "flat" + }, + { + "check": "payload_proof", + "arch": "gemma4", + "status": "SKIP", + "summary": "budget/pixel values are not observable on this platform", + "diagnosis": "This block's budgets were established without a load log (the MLX runner emits none, on either backend) and are recorded as budgets_observed = false. Nothing here is waiting on a container; the block says how the values were established." + }, + { + "check": "aspect_ladder", + "arch": "gemma4", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "gemma4", + "status": "SKIP", + "summary": "no think+format expectation recorded" + }, + { + "check": "pinned_image_token_budget", + "arch": "gemma4", + "status": "SKIP", + "summary": "no pinned-budget expectation recorded for this arch", + "diagnosis": "Omitted deliberately: the 005 defect was found on nemotron and this arch's probe has not been measured. See README.md to add one." + }, + { + "check": "text_baseline", + "arch": "qwen35moe", + "status": "PASS", + "summary": "in-image text prefix = 13 (calibrated on 256x144+512x288); text-only = 13", + "actual": 13 + }, + { + "check": "token_ladder", + "arch": "qwen35moe", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "actual": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "rows": [ + { + "size": "256x144", + "expected": 68, + "actual": 68, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 146, + "actual": 146, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "1024x576", + "expected": 578, + "actual": 578, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 5186, + "actual": 5186, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "budget/pixel values are not observable on this platform", + "diagnosis": "This block's budgets were established without a load log (the MLX runner emits none, on either backend) and are recorded as budgets_observed = false. Nothing here is waiting on a container; the block says how the values were established." + }, + { + "check": "aspect_ladder", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "no think+format expectation recorded" + }, + { + "check": "pinned_image_token_budget", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "no pinned-budget expectation recorded for this arch", + "diagnosis": "Omitted deliberately: the 005 defect was found on nemotron and this arch's probe has not been measured. See README.md to add one." + }, + { + "check": "text_baseline", + "arch": "qwen35", + "status": "PASS", + "summary": "in-image text prefix = 13 (calibrated on 256x144+512x288); text-only = 13", + "actual": 13 + }, + { + "check": "token_ladder", + "arch": "qwen35", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "actual": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "rows": [ + { + "size": "256x144", + "expected": 68, + "actual": 68, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 146, + "actual": 146, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 578, + "actual": 578, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 5186, + "actual": 5186, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "qwen35", + "status": "SKIP", + "summary": "budget/pixel values are not observable on this platform", + "diagnosis": "This block's budgets were established without a load log (the MLX runner emits none, on either backend) and are recorded as budgets_observed = false. Nothing here is waiting on a container; the block says how the values were established." + }, + { + "check": "aspect_ladder", + "arch": "qwen35", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "qwen35", + "status": "SKIP", + "summary": "no think+format expectation recorded" + }, + { + "check": "pinned_image_token_budget", + "arch": "qwen35", + "status": "SKIP", + "summary": "no pinned-budget expectation recorded for this arch", + "diagnosis": "Omitted deliberately: the 005 defect was found on nemotron and this arch's probe has not been measured. See README.md to add one." + }, + { + "check": "metal_tensor_payload", + "arch": null, + "status": "SKIP", + "summary": "does not apply on mlx-cuda" + }, + { + "check": "mlx_payload_pin", + "arch": null, + "status": "PASS", + "summary": "MLX build a59cc23 matches the measured payload (0.32.3-53-ga59cc23), from the runner's engine-init line", + "expected": "a59cc23190f1e26fabcba8693607bb4c34b2ac2b", + "actual": "0.32.3-53-ga59cc23" + }, + { + "check": "endpoint_exclusive", + "arch": null, + "status": "PASS", + "summary": "no contention detected (worst queue wait 0.0s)", + "expected": "queue wait <= 10.0s", + "actual": "0.0s", + "models_loaded": [ + "qwen3.8:27b-nvfp4" + ] + } + ], + "summary": { + "PASS": 11, + "SKIP": 18 + } +} \ No newline at end of file diff --git a/docs/maxusai/vision-suite/preflight/runs/preflight-mlx-cuda-0403-prod-g75c6d35.json b/docs/maxusai/vision-suite/preflight/runs/preflight-mlx-cuda-0403-prod-g75c6d35.json new file mode 100644 index 00000000000..160efdd0cf3 --- /dev/null +++ b/docs/maxusai/vision-suite/preflight/runs/preflight-mlx-cuda-0403-prod-g75c6d35.json @@ -0,0 +1,395 @@ +{ + "meta": { + "host": "http://127.0.0.1:11497", + "platform": "mlx-cuda", + "image_tag": "maxusai/ollama:main-75c6d350", + "started_utc": "20261013T221039", + "expectations_file": "expectations.toml", + "schema_version": 1, + "version": "0.40.2-dynres.1-17-g75c6d35", + "profile": "mlx-cuda-0-40-2", + "patchset": null, + "container": "ollama-0.40.2-dynres.1-17-g75c6d35", + "llama_cpp_build": "631109b34", + "mlx_build": "0.32.3-53-ga59cc23" + }, + "results": [ + { + "check": "version", + "arch": null, + "status": "PASS", + "summary": "0.40.2-dynres.1-17-g75c6d35 matches profile mlx-cuda-0-40-2", + "expected": "^0\\.40\\.2-dynres(\\.\\d+)?-\\d+-g[0-9a-f]{7,40}$", + "actual": "0.40.2-dynres.1-17-g75c6d35" + }, + { + "check": "payload_pin", + "arch": null, + "status": "SKIP", + "summary": "profile records no llama_cpp_build to assert against", + "diagnosis": "Add llama_cpp_build to this profile so a llama.cpp bump fails loudly instead of inheriting stale expectations. See README.md." + }, + { + "check": "toolchain_pin", + "arch": null, + "status": "SKIP", + "summary": "profile records no toolchain_build to assert against", + "diagnosis": "Add toolchain_build (e.g. \"rocm-7.2.4\" or \"cuda-13.0\") so a ROCm/CUDA bump under an unchanged payload fails loudly instead of inheriting these expectations. Platforms with no such toolchain (metal, cpu) leave it unset and keep this skip; the MLX library is pinned separately by mlx_build." + }, + { + "check": "image_tag", + "arch": null, + "status": "PASS", + "summary": "ollama-0.40.2-dynres.1-17-g75c6d35 runs maxusai/ollama:main-75c6d350", + "expected": "maxusai/ollama:main-75c6d350", + "actual": "maxusai/ollama:main-75c6d350" + }, + { + "check": "go_patch_marker", + "arch": null, + "status": "PASS", + "summary": "--image-max-tokens present in binary (1)", + "expected": 1, + "actual": 1 + }, + { + "check": "metal_tensor_host", + "arch": null, + "status": "SKIP", + "summary": "does not apply on mlx-cuda" + }, + { + "check": "metal_tensor_runtime", + "arch": null, + "status": "SKIP", + "summary": "does not apply on mlx-cuda" + }, + { + "check": "poison_probe", + "arch": null, + "status": "SKIP", + "summary": "no poison-probe expectation recorded for this profile" + }, + { + "check": "text_baseline", + "arch": "gemma4", + "status": "PASS", + "summary": "in-image text prefix = 19 (calibrated on 256x144+512x288); text-only = 19", + "actual": 19 + }, + { + "check": "token_ladder", + "arch": "gemma4", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "actual": { + "256x144": 1102, + "512x288": 1102, + "1024x576": 1102, + "2048x1152": 1102, + "3072x1728": 1102 + }, + "rows": [ + { + "size": "256x144", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "512x288", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 1102, + "actual": 1102, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "flat", + "expected_shape": "flat" + }, + { + "check": "payload_proof", + "arch": "gemma4", + "status": "SKIP", + "summary": "budget/pixel values are not observable on this platform", + "diagnosis": "This block's budgets were established without a load log (the MLX runner emits none, on either backend) and are recorded as budgets_observed = false. Nothing here is waiting on a container; the block says how the values were established." + }, + { + "check": "aspect_ladder", + "arch": "gemma4", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "gemma4", + "status": "SKIP", + "summary": "no think+format expectation recorded" + }, + { + "check": "pinned_image_token_budget", + "arch": "gemma4", + "status": "SKIP", + "summary": "no pinned-budget expectation recorded for this arch", + "diagnosis": "Omitted deliberately: the 005 defect was found on nemotron and this arch's probe has not been measured. See README.md to add one." + }, + { + "check": "text_baseline", + "arch": "qwen35moe", + "status": "PASS", + "summary": "in-image text prefix = 13 (calibrated on 256x144+512x288); text-only = 13", + "actual": 13 + }, + { + "check": "token_ladder", + "arch": "qwen35moe", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "actual": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "rows": [ + { + "size": "256x144", + "expected": 68, + "actual": 68, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "512x288", + "expected": 146, + "actual": 146, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "1024x576", + "expected": 578, + "actual": 578, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 5186, + "actual": 5186, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "budget/pixel values are not observable on this platform", + "diagnosis": "This block's budgets were established without a load log (the MLX runner emits none, on either backend) and are recorded as budgets_observed = false. Nothing here is waiting on a container; the block says how the values were established." + }, + { + "check": "aspect_ladder", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "no think+format expectation recorded" + }, + { + "check": "pinned_image_token_budget", + "arch": "qwen35moe", + "status": "SKIP", + "summary": "no pinned-budget expectation recorded for this arch", + "diagnosis": "Omitted deliberately: the 005 defect was found on nemotron and this arch's probe has not been measured. See README.md to add one." + }, + { + "check": "text_baseline", + "arch": "qwen35", + "status": "PASS", + "summary": "in-image text prefix = 13 (calibrated on 256x144+512x288); text-only = 13", + "actual": 13 + }, + { + "check": "token_ladder", + "arch": "qwen35", + "status": "PASS", + "summary": "5/5 geometries within +/-2", + "expected": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "actual": { + "256x144": 68, + "512x288": 146, + "1024x576": 578, + "2048x1152": 2306, + "3072x1728": 5186 + }, + "rows": [ + { + "size": "256x144", + "expected": 68, + "actual": 68, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "512x288", + "expected": 146, + "actual": 146, + "delta": 0, + "ok": true, + "queue_wait_s": 0.0 + }, + { + "size": "1024x576", + "expected": 578, + "actual": 578, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "2048x1152", + "expected": 2306, + "actual": 2306, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + }, + { + "size": "3072x1728", + "expected": 5186, + "actual": 5186, + "delta": 0, + "ok": true, + "queue_wait_s": 0.01 + } + ], + "shape": "scaling", + "expected_shape": "dynamic" + }, + { + "check": "payload_proof", + "arch": "qwen35", + "status": "SKIP", + "summary": "budget/pixel values are not observable on this platform", + "diagnosis": "This block's budgets were established without a load log (the MLX runner emits none, on either backend) and are recorded as budgets_observed = false. Nothing here is waiting on a container; the block says how the values were established." + }, + { + "check": "aspect_ladder", + "arch": "qwen35", + "status": "SKIP", + "summary": "no aspect-ladder expectation recorded" + }, + { + "check": "think_format", + "arch": "qwen35", + "status": "SKIP", + "summary": "no think+format expectation recorded" + }, + { + "check": "pinned_image_token_budget", + "arch": "qwen35", + "status": "SKIP", + "summary": "no pinned-budget expectation recorded for this arch", + "diagnosis": "Omitted deliberately: the 005 defect was found on nemotron and this arch's probe has not been measured. See README.md to add one." + }, + { + "check": "metal_tensor_payload", + "arch": null, + "status": "SKIP", + "summary": "does not apply on mlx-cuda" + }, + { + "check": "mlx_payload_pin", + "arch": null, + "status": "PASS", + "summary": "MLX build a59cc23 matches the measured payload (0.32.3-53-ga59cc23), from the runner's engine-init line", + "expected": "a59cc23190f1e26fabcba8693607bb4c34b2ac2b", + "actual": "0.32.3-53-ga59cc23" + }, + { + "check": "endpoint_exclusive", + "arch": null, + "status": "PASS", + "summary": "no contention detected (worst queue wait 0.0s)", + "expected": "queue wait <= 10.0s", + "actual": "0.0s", + "models_loaded": [ + "qwen3.8:27b-nvfp4", + "qwen3.6:35b-a3b-nvfp4" + ] + } + ], + "summary": { + "PASS": 11, + "SKIP": 18 + } +} \ No newline at end of file