diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36c99526b..f50cb9589 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,9 +44,14 @@ # release.yml because it requires a Linux self-hosted runner (currently only the # GB10 node used for release builds). Adding a PR-level CUDA *test* gate would # double runner cost for limited additional safety on PRs that don't touch -# CUDA-specific code paths. Compiling CUDA is a different question and three -# jobs below do it: `xla-compile` and `xla-link` at the shipped sm_121, and -# `cuda-sm70-compile` at sm_70, each on a narrow path filter. +# CUDA-specific code paths. Compiling CUDA is a different question and four +# jobs below do it: `xla-compile` and `xla-link` at the shipped sm_121, +# `cuda-blockfloat` at the same list, and `cuda-sm70-compile` at sm_70, each on a +# narrow path filter. `cuda-blockfloat` is the single exception to the sentence +# above, and a narrow one: it runs the block-float quantization tests and only +# those, because until issue #1934 no build anywhere compiled MLX's hardware +# NVFP4/MXFP4 converter, so those tests had never once exercised the path they +# are named for. name: CI @@ -333,6 +338,44 @@ jobs: - name: Check the guard still fails on undeclared and oversize binaries run: bash scripts/ci/check_binary_assets_test.sh + cuda-arch-lists: + name: CUDA architecture lists + # Deliberately not behind the `changes` filter, like `crate-versions` and + # `binary-assets`: it needs no toolchain, runs in seconds, and both defects + # it guards against are silent. + # + # MLX compiles its hardware NVFP4/MXFP4 converters + # (`cvt.rn.satfinite.e2m1x2.f32` in + # mlx/backend/cuda/quantized/nvfp4_quantize.cuh) only under an + # architecture-specific target, and the lists here are plain, so what + # supplies them is the per-source injection in + # `src/lib/mlx-cpp/CMakeLists.txt`. Remove that and the plain lists quietly + # go back to compiling the converters out of every artifact and every test, + # which is where issue #1934 started, so this asserts it is still there. + # + # The other direction is the obvious fix: putting `121a` in a list. That + # compiles every translation unit architecture-specific for a converter one + # of them reaches, costs 12.7 MB of archive and replaces the machine code of + # every decode kernel on every Blackwell host. Two tokens in a YAML `env` + # block, and no build output names the difference, so it is rejected here by + # name. So is the `f` spelling, which reads as a milder `a` and does not + # compile at all. + # + # The `cuda-blockfloat` job below asserts the same two properties on the + # compiled artifact, which is stronger, but it costs a CUDA build and is + # path-filtered. This one runs on every PR. + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Every Blackwell architecture list carries a hardware-capable entry + run: python3 scripts/ci/check_cuda_arch_lists.py + - name: Check the guard still fails on plain-only and family-specific lists + run: bash scripts/ci/check_cuda_arch_lists_test.sh + license-headers: name: license headers # Deliberately not behind the `changes` filter, like `crate-versions` and @@ -514,6 +557,10 @@ jobs: contents: read timeout-minutes: 90 env: + # What releases build for this runner's architecture. Pinned rather than + # left to auto-detection, which yields `121a` here and would compile every + # kernel architecture-specific (issue #1934), so the artifact this job + # verifies carries the same device code the published one does. MLX_CUDA_ARCHITECTURES: "121" steps: - uses: actions/checkout@v7 @@ -883,8 +930,10 @@ jobs: timeout-minutes: 120 env: # The value shipped for this runner's architecture. Auto-detection yields - # `121a`, which is numerically identical here but is not what releases - # build, so the gate compiles what ships. + # `121a`, which is not what releases build and would compile every kernel + # architecture-specific (issue #1934), so the gate compiles what ships. The + # hardware NVFP4/MXFP4 converters come from the per-source injection in + # `src/lib/mlx-cpp/CMakeLists.txt`, not from this list. MLX_CUDA_ARCHITECTURES: "121" # Every rustc lint, not just the `unused_imports` this job started with. # The narrower value existed because the XLA feature combination carried a @@ -1065,8 +1114,10 @@ jobs: timeout-minutes: 120 env: # The value shipped for this runner's architecture. Auto-detection yields - # `121a`, which is numerically identical here but is not what releases - # build, so the gate links what ships. + # `121a`, which is not what releases build and would compile every kernel + # architecture-specific (issue #1934), so the gate links what ships. The + # hardware NVFP4/MXFP4 converters come from the per-source injection in + # `src/lib/mlx-cpp/CMakeLists.txt`, not from this list. MLX_CUDA_ARCHITECTURES: "121" steps: - uses: actions/checkout@v7 @@ -1102,12 +1153,13 @@ jobs: # ============================================================================ # Every other CUDA job in this repository compiles for sm_121, and # release.yml ships `80;86;89;90a;100;120` on x86_64 and `90a;100;121` on - # aarch64. Nothing below sm_80 is compiled anywhere, so an arch-conditional - # break on Volta reaches `main` unnoticed and is found by whoever next builds - # from source on one. That is not hypothetical for this repository: epic - # #1536 adds `cc < 8` fallbacks to the quantized-matmul overlays precisely - # because the sm_80+ paths are wrong there, and every one of those branches - # is compiled only when some job asks nvcc for a pre-Ampere target. + # aarch64. Nothing below sm_80 is + # compiled anywhere, so an arch-conditional break on Volta reaches `main` + # unnoticed and is found by whoever next builds from source on one. That is + # not hypothetical for this repository: epic #1536 adds `cc < 8` fallbacks to + # the quantized-matmul overlays precisely because the sm_80+ paths are wrong + # there, and every one of those branches is compiled only when some job asks + # nvcc for a pre-Ampere target. # # Compile-only, and deliberately so. The realistic failure mode is a # compile-time one: a `__CUDA_ARCH__` guard that does not cover cc 7, a @@ -1177,9 +1229,15 @@ jobs: # code defect. Probe instead, and skip loudly. The moment a CUDA 12.x # runner joins the fleet this job starts gating for real with no edit. set -euo pipefail - NVCC=$(command -v nvcc || echo /usr/local/cuda/bin/nvcc) + # Same resolution order as the cuobjdump lookup below and as + # build.rs: CUDA_HOME, then the conventional location, then PATH. + # nvcc is not on PATH on this runner either. + NVCC="${CUDA_HOME:-/usr/local/cuda}/bin/nvcc" if [ ! -x "$NVCC" ]; then - echo "nvcc not found" >&2 + NVCC=$(command -v nvcc || true) + fi + if [ ! -x "${NVCC:-}" ]; then + echo "nvcc not found at ${CUDA_HOME:-/usr/local/cuda}/bin/nvcc or on PATH" >&2 exit 1 fi VERSION=$("$NVCC" --version | sed -n 's/.*release \([0-9.]*\).*/\1/p' | head -1) @@ -1211,19 +1269,24 @@ jobs: # architecture list failed to reach CMake would otherwise pass here # having compiled nothing for Volta. set -euo pipefail - # cuobjdump ships next to nvcc, which the build above already - # required, so resolving it from PATH and then from nvcc's directory - # covers both a CUDA toolkit on PATH and one reached only through an - # absolute nvcc. - CUOBJDUMP=$(command -v cuobjdump || true) - if [ -z "$CUOBJDUMP" ]; then - NVCC=$(command -v nvcc || true) - [ -n "$NVCC" ] && CUOBJDUMP="$(dirname "$NVCC")/cuobjdump" + # cuobjdump ships with the CUDA toolkit, next to nvcc, and neither is + # on PATH on the GB10 runner: a PATH-first lookup is what failed on + # the first run of the block-float job. Resolve the toolkit the way + # `src/lib/mlxcel-core/build.rs` resolves nvcc, CUDA_HOME first and + # then the conventional location, with PATH as a last resort rather + # than as the primary, so this reads the same toolkit that compiled + # the archive. Not found stays a hard failure: a skip here would make + # the gate pass while checking nothing. + CUDA_BIN="${CUDA_HOME:-/usr/local/cuda}/bin" + CUOBJDUMP="$CUDA_BIN/cuobjdump" + if [ ! -x "$CUOBJDUMP" ]; then + CUOBJDUMP=$(command -v cuobjdump || true) fi if [ ! -x "${CUOBJDUMP:-}" ]; then - echo "cuobjdump not found; it ships with the CUDA toolkit next to nvcc" >&2 + echo "cuobjdump not found at $CUDA_BIN/cuobjdump or on PATH; it ships with the CUDA toolkit next to nvcc" >&2 exit 1 fi + echo "cuobjdump: $CUOBJDUMP" LIB=$(find "$CARGO_TARGET_DIR/debug/build" -path '*/out/build/lib/libmlx.a' | head -1) if [ -z "$LIB" ]; then echo "libmlx.a not found under $CARGO_TARGET_DIR/debug/build" >&2 @@ -1236,3 +1299,211 @@ jobs: echo "expected sm_70 only, found: $ARCHES" >&2 exit 1 fi + + # ============================================================================ + # CUDA block-float converter (self-hosted GB10) + # ============================================================================ + # Compiles the shipped Blackwell list and then proves two different things + # about it: that MLX's hardware NVFP4/MXFP4 converter really reached the + # archive, and that the MXFP4/NVFP4 quantization tests still pass with it + # compiled in. Before issue #1934 neither held anywhere. Every list this + # repository built named Blackwell plainly, the converters in + # mlx/backend/cuda/quantized/nvfp4_quantize.cuh are gated on + # `__CUDA_ARCH_SPECIFIC__`, and so the tests that were supposed to cover + # block-float quantization exercised only the scalar CUTLASS fallback. + # + # It also proves the converter reached *only* there. The architecture lists + # stay plain and `src/lib/mlx-cpp/CMakeLists.txt` gives the extra image to + # `fp_quantize.cu` alone, so an architecture-specific image anywhere else + # means the injection was replaced by a target-level list and every kernel on + # every Blackwell host now runs different machine code. `cuda-arch-lists` + # above catches that spelling in the YAML; this catches it in the artifact, + # which also covers a list that never reaches CMake at all (build.rs + # auto-detects and falls back to `90a` when it cannot see the host). That + # trap is not hypothetical; it is the same one `cuda-sm70-compile` asserts + # against, for the same reason. + # + # Gate on SASS, never on PTX. The injection emits a cubin and no PTX, and the + # PTX that is emitted comes from the plain `compute_121` pass, which takes the + # fallback arm, so `cuobjdump --dump-ptx | grep cvt.rn.satfinite.e2m1x2` + # returns zero on a *correct* build. + # + # Path-filtered on `cuda_arch`, deliberately and with a known gap: a PR that + # changes only the Rust quantization code or its tests does not run this job, + # because a cold MLX build is measured in tens of minutes on the one + # self-hosted runner that also serves clippy for every Rust PR. What the filter does cover is everything that can change *which* + # device code is compiled, which is what the gate is about. Its own + # persistent target directory, because the architecture list is part of the + # build-script fingerprint and sharing with a differently-pinned job would + # make the two rebuild MLX against each other on every run. + cuda-blockfloat: + name: CUDA block-float converter + needs: changes + if: github.repository == 'lablup/mlxcel' && needs.changes.outputs.cuda_arch == 'true' + runs-on: GB10 + permissions: + contents: read + # Same reasoning as `xla-link` and `cuda-sm70-compile`: minutes warm, tens + # of minutes cold, on the one runner that also serves clippy for every Rust + # PR and the release build. Keyed per PR, so PRs never cancel each other, + # and `main` is exempt so a merge cannot cancel the run that proves the + # shipped architecture still carries the converter. + concurrency: + group: cuda-blockfloat-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + timeout-minutes: 180 + env: + # The list releases build for this runner's architecture. Keep it + # identical to the aarch64 release job's Blackwell half: a gate that + # compiles something else proves something else. + MLX_CUDA_ARCHITECTURES: "121" + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Use a persistent target directory + run: | + CARGO_TARGET="$HOME/.cargo-target/mlxcel-cuda-blockfloat-ci" + mkdir -p "$CARGO_TARGET" + echo "CARGO_TARGET_DIR=$CARGO_TARGET" >> $GITHUB_ENV + + - name: Build the block-float quantization tests + # `test-fast` inherits release but drops fat LTO, which is what the + # other CUDA test builds in this repository use: these tests drive MLX + # numerics, so an unoptimized build is not merely slower but + # impractically so. + run: cargo test -p mlxcel-core --profile test-fast --features cuda --lib --no-run + + - name: Verify the hardware fp4 converter is compiled, and only where intended + run: | + set -euo pipefail + # cuobjdump ships with the CUDA toolkit, next to nvcc, and neither is on PATH + # on the GB10 runner: a PATH-first lookup is what failed on this job's first + # run. Resolve the toolkit the way `src/lib/mlxcel-core/build.rs` resolves + # nvcc, CUDA_HOME first and then the conventional location, with PATH as a + # last resort rather than as the primary, so this reads the same toolkit that + # compiled the archive. Not found stays a hard failure: a skip here would make + # the gate pass while checking nothing. + CUDA_BIN="${CUDA_HOME:-/usr/local/cuda}/bin" + CUOBJDUMP="$CUDA_BIN/cuobjdump" + if [ ! -x "$CUOBJDUMP" ]; then + CUOBJDUMP=$(command -v cuobjdump || true) + fi + if [ ! -x "${CUOBJDUMP:-}" ]; then + echo "cuobjdump not found at $CUDA_BIN/cuobjdump or on PATH; it ships with the CUDA toolkit next to nvcc" >&2 + exit 1 + fi + echo "cuobjdump: $CUOBJDUMP" + + LIB=$(find "$CARGO_TARGET_DIR/test-fast/build" -path '*/out/build/lib/libmlx.a' | head -1) + if [ -z "$LIB" ]; then + echo "libmlx.a not found under $CARGO_TARGET_DIR/test-fast/build" >&2 + exit 1 + fi + echo "archive: $LIB ($(stat -c %s "$LIB") bytes)" + + # One checkout can hold several build directories, one per feature set and + # architecture list, and `head -1` above picks whichever comes first. Assert + # that the one being read was configured with the list this job pins, so the + # gate cannot pass by measuring a neighbour. + BUILD_DIR="$(dirname "$LIB")/.." + CONFIGURED=$(sed -n 's/^MLX_CUDA_ARCHITECTURES:[^=]*=//p' "$BUILD_DIR/CMakeCache.txt") + echo "configured with: $CONFIGURED" + if [ "$CONFIGURED" != "$MLX_CUDA_ARCHITECTURES" ]; then + echo "archive was configured with [$CONFIGURED], not the job's [$MLX_CUDA_ARCHITECTURES]" >&2 + exit 1 + fi + + WORK="${RUNNER_TEMP:-/tmp}/blockfloat-$$" + mkdir -p "$WORK" + ( cd "$WORK" && ar x "$LIB" fp_quantize.cu.o qmv.cu.o fp_qmv.cu.o ) + for obj in fp_quantize.cu.o qmv.cu.o fp_qmv.cu.o; do + if [ ! -f "$WORK/$obj" ]; then + echo "$obj not found in $LIB; MLX's quantized backend did not build as expected" >&2 + exit 1 + fi + done + + # 1. The converter is compiled. Zero means the architecture-specific image + # never reached nvcc and MLX compiled the scalar fallback, which is the + # state issue #1934 found in every artifact this project shipped. + # + # Scoped to the one translation unit that can carry the instruction rather + # than to the whole archive: `--dump-sass` on the 178 MB archive takes about + # ten minutes and this takes under a second, for the same verdict. + "$CUOBJDUMP" --dump-sass "$WORK/fp_quantize.cu.o" > "$WORK/sass.txt" + FP4=$(grep -c 'F2FP.SATFINITE.E2M1' "$WORK/sass.txt" || true) + echo "F2FP.SATFINITE.E2M1 occurrences in fp_quantize.cu.o: $FP4" + if [ "$FP4" -eq 0 ]; then + echo "no hardware fp4 conversion instruction in libmlx.a: MLX compiled the scalar fallback" >&2 + exit 1 + fi + # The fp8 converter is not gated on an architecture-specific target, so it is + # present either way. Printing it beside the fp4 count makes "nothing compiled + # at all" read differently from "the fp4 gate is off". + echo "F2FP.SATFINITE.E4M3 occurrences (ungated control): $(grep -c 'F2FP.SATFINITE.E4M3' "$WORK/sass.txt" || true)" + + # 2. The converter is compiled ONLY there. This is the half that separates the + # per-source injection from naming Blackwell `121a` in the architecture + # list. The list form gives every object an architecture-specific image, and + # on a matching device that is the image the driver loads, so every kernel + # on every Blackwell host would run different machine code than it does + # today. Decode is supposed to be untouched by a converter fix. + for obj in qmv.cu.o fp_qmv.cu.o; do + IMAGES=$("$CUOBJDUMP" --list-elf "$WORK/$obj" | grep -oE 'sm_[0-9]+[af]?' | sort -u | tr '\n' ' ') + echo "$obj images: $IMAGES" + if printf '%s' "$IMAGES" | grep -qE 'sm_[0-9]+[af]'; then + echo "$obj carries an architecture-specific image; decode was not supposed to change (issue #1934)" >&2 + exit 1 + fi + done + ARCH_SPECIFIC=$("$CUOBJDUMP" --list-elf "$LIB" | grep -cE 'sm_[0-9]+a' || true) + echo "architecture-specific cubin images in the whole archive: $ARCH_SPECIFIC" + if [ "$ARCH_SPECIFIC" -ne 1 ]; then + echo "expected exactly one architecture-specific image in libmlx.a, the one in fp_quantize.cu.o, found $ARCH_SPECIFIC" >&2 + exit 1 + fi + + # 3. Forward compatibility is untouched. Plain `compute_121` PTX is what a + # later Blackwell revision JITs from, and the injection emits a cubin only, + # so this count must not move. + PTX=$("$CUOBJDUMP" --dump-ptx "$LIB" | grep -cE '^\.target sm_121$' || true) + echo "forward-JIT-capable plain PTX images (.target sm_121): $PTX" + if [ "$PTX" -eq 0 ]; then + echo "no plain compute_121 PTX in libmlx.a; the architecture list is not what this job pinned" >&2 + exit 1 + fi + + # 4. The assumption the per-source injection rests on: `nvfp4_quantize.cuh` is + # the only place in MLX that gates code on an architecture-specific target. + # If a pin bump adds another, a quantized GEMM say, the injection silently + # stops covering the arch-specific surface, and this is the only place that + # would notice. + MLX_SRC="$BUILD_DIR/_deps/mlx-src/mlx" + if [ ! -d "$MLX_SRC" ]; then + echo "MLX source tree not found at $MLX_SRC; cannot check the arch-specific surface" >&2 + exit 1 + fi + GATED=$(grep -rl '__CUDA_ARCH_SPECIFIC__\|__CUDA_ARCH_FAMILY_SPECIFIC__' "$MLX_SRC" | sed "s|$MLX_SRC/||" | sort | tr '\n' ' ' | sed 's/ $//') + echo "MLX files gated on an architecture-specific target: $GATED" + if [ "$GATED" != "backend/cuda/quantized/nvfp4_quantize.cuh" ]; then + echo "the arch-specific surface of the pinned MLX has changed; the per-source injection in src/lib/mlx-cpp/CMakeLists.txt covers only fp_quantize.cu, so decide whether to extend it or move to a target-level architecture list (issue #1934)" >&2 + exit 1 + fi + rm -rf "$WORK" + + - name: Run the MXFP4 and NVFP4 quantization tests + # `global_scale` selects the six `ffi_tests` cases that drive + # `quantize_weights_with_mode` with `mxfp4` and native NVFP4 sidecars, + # which is every test in this crate that reaches MLX's block-float + # quantization kernels. `shapeless_compile_audit_harness` touches them + # too but carries `#[ignore]` for a hardware audit script, so naming it + # here would add a skipped line and no coverage. + # + # `--test-threads=1` is mandatory for CUDA on this runner: parallel MLX + # tests abort regardless of the change under test. + run: | + cargo test -p mlxcel-core --profile test-fast --features cuda --lib -- \ + --test-threads=1 \ + global_scale diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 107038bc4..c52e05d30 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -537,13 +537,25 @@ jobs: # 90a is load-bearing for MLX's Hopper quantized kernel gate, and build.rs # forwards the list verbatim. The CLI and server (~347 MB each) ship as # separate archives, both with the CCCL headers. + # + # Blackwell stays plain here even though MLX's hardware NVFP4/MXFP4 + # converters are gated on `__CUDA_ARCH_SPECIFIC__` and a plain target + # compiles them out (issue #1934). Naming Blackwell `a` in this list + # would enable them for every translation unit, which costs 12.7 MB of + # archive and changes the machine code of every kernel, decode included. + # `src/lib/mlx-cpp/CMakeLists.txt` adds the architecture-specific image to + # `fp_quantize.cu` alone instead, which is the only translation unit that + # can reach those converters, so this list needs no `a` entry to get them. + # scripts/ci/check_cuda_arch_lists.py rejects one. env: MLX_CUDA_ARCHITECTURES: "90a;100;121" CLI_ASSET: mlxcel-linux-aarch64-cuda13 SERVER_ASSET: mlxcel-server-linux-aarch64-cuda13 - # Cold build compiles MLX for three CUDA architectures; generous ceiling so - # a from-scratch build is never killed (the cargo cache makes reruns cheap). + # Cold build compiles MLX for three CUDA architectures, plus one extra + # code-generation pass for `fp_quantize.cu` per Blackwell entry; generous + # ceiling so a from-scratch build is never killed (the cargo cache makes + # reruns cheap). timeout-minutes: 360 steps: @@ -756,10 +768,16 @@ jobs: # One fat binary covering Ampere and later. 90a is load-bearing: MLX gates # its Hopper quantized kernel on "90a" being in the list (MLX_CUDA_SM90A_ENABLED), - # and build.rs forwards MLX_CUDA_ARCHITECTURES verbatim (no auto-suffix). Blackwell - # (sm_100/sm_120) has no MLX gate, so plain numbers emit SASS+PTX. sm_87 (Jetson) - # and sm_121 (GB10) are aarch64-only and excluded here. The build runner's own GPU - # is an RTX 5060 (sm_120), so the smoke test below runs on native SASS. + # and build.rs forwards MLX_CUDA_ARCHITECTURES verbatim (no auto-suffix). + # Blackwell stays plain for the reason spelled out in the aarch64 job + # above: the hardware NVFP4/MXFP4 converters are added to `fp_quantize.cu` + # alone by `src/lib/mlx-cpp/CMakeLists.txt` rather than to the whole target. + # This corrects the rationale that stood here before, which said Blackwell + # had no MLX gate and that plain numbers were therefore enough. There is a + # gate (issue #1934); it is simply not answered from this list. sm_87 + # (Jetson) and sm_121 (GB10) are aarch64-only and excluded here. The build + # runner's own GPU is an RTX 5060 (sm_120), so the smoke test below runs on + # native SASS. env: MLX_CUDA_ARCHITECTURES: "80;86;89;90a;100;120" # The CLI and server binaries are ~347 MB each, so ship them as separate @@ -768,9 +786,10 @@ jobs: SERVER_ASSET: mlxcel-server-linux-x86_64-cuda13 # A cold build compiles MLX for six CUDA architectures (CUTLASS-heavy - # quantized kernels per arch); the persistent cargo cache makes later runs - # far cheaper. Generous ceiling so a from-scratch build is not killed (a - # cold 6-arch build was observed at ~3h). + # quantized kernels per arch), plus one extra pass for `fp_quantize.cu` per + # Blackwell entry; the persistent cargo cache makes later runs far cheaper. + # Generous ceiling so a from-scratch build is not killed (a cold six-arch + # build was observed at ~3h). timeout-minutes: 480 steps: diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/README.md b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/README.md new file mode 100644 index 000000000..4f2bafc36 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/README.md @@ -0,0 +1,59 @@ +# Does compiling MLX architecture-specific cost decode? (GB10, 2026-09-21) + +Issue #1934. Host: NVIDIA GB10 (sm_121, DGX Spark), Linux aarch64 `7.0.0-1019-nvidia`, CUDA 13.0.88, MLX pin `81ba1c6a`, toolchain 1.97.1. Source tree `01fe5bbc` (this PR's branch, rebased onto `76c849b5`). Pairing: `models/mlx/qwen3.5-4b-4bit` with `models/mlx/qwen3.5-4b-dflash`, draft width 4 and classic. Driver: `harness/ab_interleave.py`, which imports `run_arm` from the #1797 served harness (`data/draft-block-width-gb10-2026-09-20/harness/`) rather than reimplementing the host gate, the NVRM trip wire and the DFlash diagnostics parsing. + +## Why interleaved + +A sequential A/B on this pairing reported the architecture-specific build about 2.6% slower at width 4 with disjoint ranges. Two arms measured one after the other are only as comparable as the host is still between them, and on this box that is the thing most likely to be false. This driver alternates at round granularity instead: one server start, one measured request, teardown, next binary, and the arm that opens rotates every round, so no binary systematically owns the warm or the cool half of the session. + +## Arms + +All three binaries are built from the same source tree and differ only in how MLX was compiled. + +| arm | `MLX_CUDA_ARCHITECTURES` | `libmlx.a` | cubin images in the binary | +|---|---|---|---| +| `121` | `121` | 178,388,958 B | 95 `sm_121` | +| `121a-real;121` | `121a-real;121` | 191,110,534 B | 95 `sm_121`, 95 `sm_121a` | +| `121+fp_quantize-a` | `121`, plus the per-source injection | 178,438,494 B | 95 `sm_121`, 1 `sm_121a` | + +The third arm is also the experiment's negative control. Its decode kernels are compiled with byte-identical flags to the first arm's, and their disassembly hashes match (`qmv.cu.o` sm_121 section SHA-256 `6eb4cf943169` in both, 3,640,476 lines; `fp_qmv.cu.o` `53faf7a4e1d5`, 65,040 lines), with no second image for the driver to prefer. Any difference it shows cannot be a decode-kernel-speed difference. + +## Result: no decode cost survives interleaving + +30 arms, 0 errors, `nvrm_delta` 0 everywhere, no CI job overlapping any arm, load1 before each arm between 0.01 and 0.47. All fifteen width-4 runs produced byte-identical text from a 158-token prompt, across all three binaries, so the arms did the same work and what is being compared is how fast they did it. + +| width | arm | n | min | max | mean | ratio to `121` | ranges disjoint | +|---|---|---|---|---|---|---|---| +| 4 | `121` | 5 | 65.96 | 67.81 | 66.97 | | | +| 4 | `121a-real;121` | 5 | 66.04 | 68.18 | 67.31 | 1.0052 | no | +| 4 | `121+fp_quantize-a` | 5 | 65.40 | 67.17 | 66.21 | 0.9887 | no | +| classic | `121` | 5 | 55.72 | 59.39 | 57.21 | | | +| classic | `121a-real;121` | 5 | 55.94 | 57.24 | 56.76 | 0.9923 | no | +| classic | `121+fp_quantize-a` | 5 | 55.09 | 57.25 | 56.34 | 0.9849 | no | + +Paired within-round differences against `121`, which removes round-level drift: + +| width | arm | deltas (tok/s) | mean | +|---|---|---|---| +| 4 | `121a-real;121` | +0.81, +1.67, +1.54, -0.60, -1.67 | +0.54% | +| 4 | `121+fp_quantize-a` | -0.26, -1.10, +1.21, -1.55, -2.08 | -1.11% | +| classic | `121a-real;121` | -1.24, +0.22, -0.09, +1.13, -2.22 | -0.73% | +| classic | `121+fp_quantize-a` | -1.18, -0.63, -0.31, -0.05, -2.14 | -1.48% | + +The architecture-specific arm is not slower. At width 4 it is nominally faster, its paired deltas change sign across rounds, and every range overlaps. + +What bounds the method is the negative control. `121+fp_quantize-a` runs decode kernels that are the same bytes as `121`, and it measures 1.11% slower at width 4 and 1.48% slower at classic with all five paired deltas negative. So five same-sign deltas at about 1.5% are reachable here with no code difference at all, which is what the earlier 2.6% sequential result has to be read against. + +## Reproducing + +```bash +MLXCEL_SWEEP_HARNESS=/docs/benchmark_results/data/draft-block-width-gb10-2026-09-20/harness \ +python3 harness/ab_interleave.py \ + --arm "121=" \ + --arm "121a-real;121=" \ + --arm "121+fp_quantize-a=" \ + --target /qwen3.5-4b-4bit --drafter /qwen3.5-4b-dflash \ + --rounds 5 --widths 4,classic --n 1 --out ab.jsonl --logdir logs +``` + +`ab.jsonl` is one record per arm, carrying the host-gate readings, the server command line, the DFlash diagnostics and the generated text. `logs/` holds each arm's server log; the `CUDA compute capability ...; compiled for [...]` startup line in each one names the architecture list that binary was compiled for, which is how an arm's label is checked against the binary rather than trusted. diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/ab.jsonl b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/ab.jsonl new file mode 100644 index 000000000..164033e75 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/ab.jsonl @@ -0,0 +1,30 @@ +{"arm": "r1.A.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.4677734375, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 116.7, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r1.A.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.56, "warmup_chars": 820, "runs": [{"wall_s": 2.992, "e2e_tok_s": 66.838, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2244.517595999999, "verify_ms": 178.152222, "target_argmax_sync_ms": 2492.190311999999, "decode_ms": 4967.833798}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 269.96634900000004, "verify_ms": 183.84626500000002, "target_argmax_sync_ms": 2360.524236, "decode_ms": 2864.3033889999997}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 1, "side": "A", "arch_list": "121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "opened_round": true} +{"arm": "r1.B.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.26025390625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r1.B.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.379, "warmup_chars": 820, "runs": [{"wall_s": 2.957, "e2e_tok_s": 67.644, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2117.74567, "verify_ms": 184.213324, "target_argmax_sync_ms": 2437.9943560000015, "decode_ms": 4788.528716}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 246.11965800000002, "verify_ms": 183.91523999999998, "target_argmax_sync_ms": 2350.798859000001, "decode_ms": 2829.950116}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 1, "side": "B", "arch_list": "121a-real;121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "opened_round": false} +{"arm": "r1.C.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.17919921875, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r1.C.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.359, "warmup_chars": 820, "runs": [{"wall_s": 3.004, "e2e_tok_s": 66.579, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2052.1914949999996, "verify_ms": 185.70774500000002, "target_argmax_sync_ms": 2473.857789, "decode_ms": 4762.442584}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 267.97565699999996, "verify_ms": 183.81967099999994, "target_argmax_sync_ms": 2376.9887470000003, "decode_ms": 2878.692496}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 1, "side": "C", "arch_list": "121+fp_quantize-a", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "opened_round": false} +{"arm": "r1.A.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.26025390625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r1.A.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 3.958, "warmup_chars": 823, "runs": [{"wall_s": 3.466, "e2e_tok_s": 57.698, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 1, "side": "A", "arch_list": "121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "opened_round": true} +{"arm": "r1.B.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.13525390625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.1, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r1.B.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 4.012, "warmup_chars": 823, "runs": [{"wall_s": 3.542, "e2e_tok_s": 56.462, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 1, "side": "B", "arch_list": "121a-real;121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "opened_round": false} +{"arm": "r1.C.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.17919921875, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r1.C.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 3.993, "warmup_chars": 823, "runs": [{"wall_s": 3.539, "e2e_tok_s": 56.517, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 1, "side": "C", "arch_list": "121+fp_quantize-a", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "opened_round": false} +{"arm": "r2.B.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.1103515625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r2.B.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.381, "warmup_chars": 820, "runs": [{"wall_s": 2.933, "e2e_tok_s": 68.181, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2123.2643770000013, "verify_ms": 176.516361, "target_argmax_sync_ms": 2434.534705, "decode_ms": 4789.819776}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 258.337034, "verify_ms": 184.72328, "target_argmax_sync_ms": 2317.547057999999, "decode_ms": 2809.8846970000004}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 2, "side": "B", "arch_list": "121a-real;121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "opened_round": true} +{"arm": "r2.C.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.28173828125, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r2.C.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.52, "warmup_chars": 820, "runs": [{"wall_s": 3.058, "e2e_tok_s": 65.404, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2160.3745980000003, "verify_ms": 180.86351299999998, "target_argmax_sync_ms": 2516.9336369999996, "decode_ms": 4909.299134}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 276.4062280000001, "verify_ms": 173.241899, "target_argmax_sync_ms": 2427.1663460000004, "decode_ms": 2926.59653}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 2, "side": "C", "arch_list": "121+fp_quantize-a", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "opened_round": false} +{"arm": "r2.A.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.162109375, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r2.A.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.389, "warmup_chars": 820, "runs": [{"wall_s": 3.007, "e2e_tok_s": 66.507, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2125.874181999999, "verify_ms": 175.27199600000003, "target_argmax_sync_ms": 2433.895492, "decode_ms": 4785.179344}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 272.8918859999999, "verify_ms": 178.38529700000004, "target_argmax_sync_ms": 2382.083207, "decode_ms": 2883.853979}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 2, "side": "A", "arch_list": "121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "opened_round": false} +{"arm": "r2.B.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.07763671875, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r2.B.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 4.018, "warmup_chars": 823, "runs": [{"wall_s": 3.575, "e2e_tok_s": 55.94, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 2, "side": "B", "arch_list": "121a-real;121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "opened_round": true} +{"arm": "r2.C.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.04541015625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r2.C.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 4.024, "warmup_chars": 823, "runs": [{"wall_s": 3.63, "e2e_tok_s": 55.09, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 2, "side": "C", "arch_list": "121+fp_quantize-a", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "opened_round": false} +{"arm": "r2.A.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.01220703125, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r2.A.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 4.067, "warmup_chars": 823, "runs": [{"wall_s": 3.589, "e2e_tok_s": 55.72, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 2, "side": "A", "arch_list": "121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "opened_round": false} +{"arm": "r3.C.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.19677734375, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r3.C.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.324, "warmup_chars": 820, "runs": [{"wall_s": 2.977, "e2e_tok_s": 67.171, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2057.86943, "verify_ms": 178.767745, "target_argmax_sync_ms": 2440.3703560000004, "decode_ms": 4731.529863000001}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 254.99693900000008, "verify_ms": 183.19217499999993, "target_argmax_sync_ms": 2356.811484, "decode_ms": 2848.891869}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 3, "side": "C", "arch_list": "121+fp_quantize-a", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "opened_round": true} +{"arm": "r3.A.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.31640625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r3.A.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.367, "warmup_chars": 820, "runs": [{"wall_s": 3.032, "e2e_tok_s": 65.962, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2060.6090190000004, "verify_ms": 178.89819000000003, "target_argmax_sync_ms": 2482.075442, "decode_ms": 4773.435041}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 268.06839699999995, "verify_ms": 189.78366400000002, "target_argmax_sync_ms": 2392.2849709999996, "decode_ms": 2905.187327}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 3, "side": "A", "arch_list": "121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "opened_round": false} +{"arm": "r3.B.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.29345703125, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r3.B.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.402, "warmup_chars": 820, "runs": [{"wall_s": 2.963, "e2e_tok_s": 67.5, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2124.335785999999, "verify_ms": 183.52257600000002, "target_argmax_sync_ms": 2436.8887019999997, "decode_ms": 4795.475762}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 266.3839079999999, "verify_ms": 182.06360999999995, "target_argmax_sync_ms": 2338.1235519999996, "decode_ms": 2835.84458}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 3, "side": "B", "arch_list": "121a-real;121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "opened_round": false} +{"arm": "r3.C.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.46728515625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r3.C.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 3.984, "warmup_chars": 823, "runs": [{"wall_s": 3.507, "e2e_tok_s": 57.024, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 3, "side": "C", "arch_list": "121+fp_quantize-a", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "opened_round": true} +{"arm": "r3.A.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.2724609375, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r3.A.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 3.984, "warmup_chars": 823, "runs": [{"wall_s": 3.488, "e2e_tok_s": 57.333, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 3, "side": "A", "arch_list": "121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "opened_round": false} +{"arm": "r3.B.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.13916015625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r3.B.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 3.964, "warmup_chars": 823, "runs": [{"wall_s": 3.494, "e2e_tok_s": 57.239, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 3, "side": "B", "arch_list": "121a-real;121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "opened_round": false} +{"arm": "r4.A.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.123046875, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r4.A.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.377, "warmup_chars": 820, "runs": [{"wall_s": 2.949, "e2e_tok_s": 67.809, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2118.4096140000006, "verify_ms": 175.59244000000004, "target_argmax_sync_ms": 2433.411687, "decode_ms": 4774.0627}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 259.74100899999996, "verify_ms": 179.4162, "target_argmax_sync_ms": 2337.9900299999995, "decode_ms": 2822.461017}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 4, "side": "A", "arch_list": "121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "opened_round": true} +{"arm": "r4.B.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.20556640625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r4.B.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.406, "warmup_chars": 820, "runs": [{"wall_s": 2.976, "e2e_tok_s": 67.206, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2125.8520889999995, "verify_ms": 187.34871900000002, "target_argmax_sync_ms": 2438.0010700000003, "decode_ms": 4802.527433}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 258.218033, "verify_ms": 184.78606400000004, "target_argmax_sync_ms": 2357.277712000001, "decode_ms": 2850.464603}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 4, "side": "B", "arch_list": "121a-real;121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "opened_round": false} +{"arm": "r4.C.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.1572265625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r4.C.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.36, "warmup_chars": 820, "runs": [{"wall_s": 3.019, "e2e_tok_s": 66.257, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2076.686953, "verify_ms": 177.15159699999998, "target_argmax_sync_ms": 2460.0480639999996, "decode_ms": 4765.233806}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 251.316414, "verify_ms": 183.31045099999997, "target_argmax_sync_ms": 2405.8784010000013, "decode_ms": 2891.754367}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 4, "side": "C", "arch_list": "121+fp_quantize-a", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "opened_round": false} +{"arm": "r4.A.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.09765625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r4.A.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 4.044, "warmup_chars": 823, "runs": [{"wall_s": 3.579, "e2e_tok_s": 55.888, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 4, "side": "A", "arch_list": "121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "opened_round": true} +{"arm": "r4.B.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.0556640625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r4.B.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 3.983, "warmup_chars": 823, "runs": [{"wall_s": 3.508, "e2e_tok_s": 57.014, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 4, "side": "B", "arch_list": "121a-real;121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "opened_round": false} +{"arm": "r4.C.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.083984375, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r4.C.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 3.981, "warmup_chars": 823, "runs": [{"wall_s": 3.582, "e2e_tok_s": 55.84, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 4, "side": "C", "arch_list": "121+fp_quantize-a", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "opened_round": false} +{"arm": "r5.B.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.15283203125, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r5.B.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.396, "warmup_chars": 820, "runs": [{"wall_s": 3.029, "e2e_tok_s": 66.037, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2106.660882, "verify_ms": 174.18058799999994, "target_argmax_sync_ms": 2475.014844, "decode_ms": 4807.330501}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 262.36315399999995, "verify_ms": 184.79203900000005, "target_argmax_sync_ms": 2407.873518, "decode_ms": 2903.505671}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 5, "side": "B", "arch_list": "121a-real;121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "opened_round": true} +{"arm": "r5.C.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.146484375, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r5.C.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.366, "warmup_chars": 820, "runs": [{"wall_s": 3.047, "e2e_tok_s": 65.636, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2069.6160390000005, "verify_ms": 187.899667, "target_argmax_sync_ms": 2462.2250029999996, "decode_ms": 4771.318085}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 264.33038100000005, "verify_ms": 185.29048299999994, "target_argmax_sync_ms": 2418.631441, "decode_ms": 2919.552205}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 5, "side": "C", "arch_list": "121+fp_quantize-a", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "opened_round": false} +{"arm": "r5.A.w4", "width": 4, "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1", "--draft-model", "/home/inureyes/models/mlx/qwen3.5-4b-dflash", "--draft-kind", "dflash", "--draft-block-size", "4"], "arm_env": {}, "load1_before": 0.14697265625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r5.A.w4.log", "startup_s": 4.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 5.39, "warmup_chars": 820, "runs": [{"wall_s": 2.954, "e2e_tok_s": 67.711, "chunks": 200, "chars": 820, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n logger: logging.Logger = None,\n **kwargs,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n Args:\n func: The function to retry.\n backoff_policy: The backoff policy to use.\n logger: The logger to use for logging.\n **kwargs: Additional arguments to pass to the function.\n\n Returns:\n The result of the function.\n \"\"\"\n if"}], "speculative_line": "speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)", "diagnostics": [{"block_size": 4, "rounds": 0, "proposed_tokens": 0, "accepted_tokens": 0, "acceptance_rate": 0.0, "emitted_per_verify": 0.0, "draft_ms": 0.0, "verify_ms": 0.0, "target_argmax_sync_ms": 0.0, "decode_ms": 0.0}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 2119.0758200000005, "verify_ms": 190.83996199999999, "target_argmax_sync_ms": 2426.652814, "decode_ms": 4787.517066}, {"block_size": 4, "rounds": 79, "proposed_tokens": 237, "accepted_tokens": 121, "acceptance_rate": 0.510548523206751, "emitted_per_verify": 2.518987341772152, "draft_ms": 261.52825899999993, "verify_ms": 182.45939100000007, "target_argmax_sync_ms": 2333.550974, "decode_ms": 2827.54142}], "diagnostics_block_sizes": [4], "diagnostics_rounds": [0, 79], "declined_to_classic": false, "speculative_ran": true, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 5, "side": "A", "arch_list": "121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "opened_round": false} +{"arm": "r5.B.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.09228515625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r5.B.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 3.984, "warmup_chars": 823, "runs": [{"wall_s": 3.498, "e2e_tok_s": 57.169, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 5, "side": "B", "arch_list": "121a-real;121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-combined", "opened_round": true} +{"arm": "r5.C.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.1181640625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.2, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r5.C.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 4.005, "warmup_chars": 823, "runs": [{"wall_s": 3.493, "e2e_tok_s": 57.25, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 5, "side": "C", "arch_list": "121+fp_quantize-a", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-pertu", "opened_round": false} +{"arm": "r5.A.wclassic", "width": "classic", "cmd": ["/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "-m", "/home/inureyes/models/mlx/qwen3.5-4b-4bit", "--port", "18934", "--ignore-eos", "--max-batch-size", "1"], "arm_env": {}, "load1_before": 0.1572265625, "ci_job_running": false, "gate_wait_s": 0, "driver_wait_s": 0, "mem_wait_s": 0, "mem_available_gib_before": 117.1, "nvrm_window_before": 0, "nvrm_total_before": 0, "log": "/tmp/claude-1000/-home-inureyes-Development-mlxcel/a7ac83cc-0ca7-4ae0-8f19-a24a07471f2a/scratchpad/ab-run/server.r5.A.wclassic.log", "startup_s": 3.0, "model_id": "qwen3.5-4b-4bit", "prompt_tokens": 158, "warmup_s": 3.879, "warmup_chars": 823, "runs": [{"wall_s": 3.368, "e2e_tok_s": 59.389, "chunks": 200, "chars": 823, "completion_tokens": 200, "text": " \"\"\"Generate a sequence of delays for retrying.\"\"\"\n delays = []\n for i in range(self.max_attempts):\n delay = self.base_delay * (2 ** i)\n delay = min(delay, self.max_delay)\n if self.jitter:\n delay = delay + random.uniform(0, delay)\n delays.append(delay)\n return delays\n\n\ndef retry_with_backoff(\n func,\n backoff_policy: BackoffPolicy = None,\n *,\n logger: logging.Logger = None,\n max_attempts: int = None,\n max_delay: float = None,\n base_delay: float = None,\n jitter: bool = None,\n):\n \"\"\"Retry a function with exponential backoff and jitter.\n\n This decorator is useful for retrying HTTP calls that might fail due to\n transient network issues. It uses exponential backoff with jitter to\n avoid thundering"}], "speculative_line": null, "diagnostics": [], "diagnostics_block_sizes": [], "diagnostics_rounds": [], "declined_to_classic": false, "speculative_ran": false, "nvrm_total_after": 0, "nvrm_delta": 0, "nvrm_window_after": 0, "round": 5, "side": "A", "arch_list": "121", "server_bin": "/home/inureyes/.cargo-target/issue-1934-artifacts/mlxcel-server-121", "opened_round": false} diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/harness/ab_interleave.py b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/harness/ab_interleave.py new file mode 100644 index 000000000..81d20f887 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/harness/ab_interleave.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Interleaved two-binary A/B for issue #1934, built on #1797's served sweep. + +The question is whether compiling MLX for `121a-real;121` instead of `121` +costs decode throughput, because the architecture-specific cubin is the image +the driver loads and CUTLASS compiles the decode kernels differently under it +(`CUTLASS_ARCH_MMA_SM121A_ENABLED`, derived from `__CUDA_ARCH_SPECIFIC__`). + +A sequential A/B answers that only as well as the host holds still between the +two arms. This driver removes that dependency by interleaving at round +granularity: one server start, one measured request, teardown, then the other +binary, and the arm that opens alternates every round so neither binary +systematically owns the warmer or cooler half of the session. + +`run_arm` is imported from the #1797 harness rather than reimplemented, so the +host gate, the NVRM trip wire, the DFlash diagnostics parsing and the +"speculative actually ran" predicate are the same code that produced the +records this is compared against. + + ab_interleave.py --a-bin BIN_121 --b-bin BIN_COMBINED \\ + --target DIR --drafter DIR --rounds 6 --out ab.jsonl +""" +import argparse +import json +import os +import statistics +import sys + +# The #1797 served harness, imported rather than copied. Point +# MLXCEL_SWEEP_HARNESS at it; there is no useful default, because this driver +# is meant to run from anywhere and guessing a repository root from __file__ +# only produces a confusing failure two imports later. +HARNESS = os.environ.get("MLXCEL_SWEEP_HARNESS") +if not HARNESS or not os.path.isdir(HARNESS): + raise SystemExit( + "set MLXCEL_SWEEP_HARNESS to the #1797 harness directory " + "(docs/benchmark_results/data/draft-block-width-gb10-2026-09-20/harness)" + ) +sys.path.insert(0, HARNESS) +import sweep_server_widths as sweep # noqa: E402 + + +class Args: + """The attribute surface `sweep.run_arm` reads off its argparse namespace.""" + + def __init__(self, **kw): + self.__dict__.update(kw) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "--arm", + action="append", + required=True, + metavar="NAME=PATH", + help="one server binary to compare, repeatable; NAME labels it in the record", + ) + ap.add_argument("--target", required=True) + ap.add_argument("--drafter", required=True) + ap.add_argument("--rounds", type=int, default=6) + ap.add_argument( + "--widths", + default="4,classic", + help="arms measured per binary per round; `classic` drops the drafter", + ) + ap.add_argument("--n", type=int, default=1, help="measured requests per server start") + ap.add_argument("--max-tokens", type=int, default=200) + ap.add_argument("--port", type=int, default=18934) + ap.add_argument("--out", required=True) + ap.add_argument("--logdir", default=None) + a = ap.parse_args() + + a.logdir = a.logdir or os.path.dirname(os.path.abspath(a.out)) + os.makedirs(a.logdir, exist_ok=True) + prompt = open(os.path.join(HARNESS, "prompt_retry.txt")).read() + widths = [w if w == "classic" else int(w) for w in a.widths.split(",")] + + arms = [] + for spec in a.arm: + name, _, path = spec.partition("=") + if not path or not os.path.exists(path): + raise SystemExit(f"--arm {spec}: expected NAME=PATH with an existing binary") + arms.append((chr(ord("A") + len(arms)), path, name)) + out = open(a.out, "w") + records = [] + + for rnd in range(1, a.rounds + 1): + # Rotate which binary opens the round. Whatever the host does over the + # session, each binary spends an equal share of its runs in the leading + # slot, which is the slot that pays for anything the previous arm left + # warm or cold. + shift = (rnd - 1) % len(arms) + order = arms[shift:] + arms[:shift] + for width in widths: + for side, binary, name in order: + args = Args( + server=binary, + target=a.target, + drafter=a.drafter, + port=a.port, + n=a.n, + max_tokens=a.max_tokens, + logdir=a.logdir, + ) + tag = f"r{rnd}.{side}.w{width}" + # `run_arm` does `env.setdefault("MLX_CUDA_ARCHITECTURES", "121")` + # and writes the environment into the arm's server log. The + # variable is build-time only and inert here, but leaving the + # default in place would put "121" in the combined arm's log and + # make the record contradict itself. Set the arm's real value so + # the setdefault is a no-op. + os.environ["MLX_CUDA_ARCHITECTURES"] = name + rec = sweep.run_arm(args, width, prompt, tag) + rec["round"] = rnd + rec["side"] = side + rec["arch_list"] = name + rec["server_bin"] = binary + rec["opened_round"] = side == order[0][0] + records.append(rec) + out.write(json.dumps(rec) + "\n") + out.flush() + runs = rec.get("runs") or [] + toks = [r["e2e_tok_s"] for r in runs] + print( + f"[{tag}] {name}: " + + (", ".join(f"{t:.2f}" for t in toks) if toks else rec.get("error", "no runs")) + + f" spec_ran={rec.get('speculative_ran')}" + + f" nvrm_delta={rec.get('nvrm_delta')}" + + f" ci={rec.get('ci_job_running')}", + flush=True, + ) + out.close() + + print("\n=== summary ===") + for width in widths: + print(f"-- width {width} --") + for _, _, name in arms: + vals = [ + r["e2e_tok_s"] + for rec in records + if rec["arch_list"] == name and rec["width"] == width + for r in (rec.get("runs") or []) + ] + if not vals: + print(f" {name}: no runs") + continue + print( + f" {name}: n={len(vals)} min={min(vals):.2f} max={max(vals):.2f} " + f"mean={statistics.mean(vals):.2f} " + f"median={statistics.median(vals):.2f}" + ) + base_name = arms[0][2] + base = [ + r["e2e_tok_s"] + for rec in records + if rec["arch_list"] == base_name and rec["width"] == width + for r in (rec.get("runs") or []) + ] + for _, _, name in arms[1:]: + vals = [ + r["e2e_tok_s"] + for rec in records + if rec["arch_list"] == name and rec["width"] == width + for r in (rec.get("runs") or []) + ] + if not (base and vals): + continue + disjoint = max(vals) < min(base) or max(base) < min(vals) + print( + f" {name} / {base_name}: " + f"ratio of means {statistics.mean(vals) / statistics.mean(base):.4f}, " + f"ranges disjoint {disjoint}" + ) + + +if __name__ == "__main__": + main() diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.A.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.A.w4.log new file mode 100644 index 000000000..c8abd2bb4 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.A.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:20:15.437876Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:20:15.437966Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:20:15.438412Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:20:15.438515Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:20:15.438517Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:20:15.438519Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:20:15.783834Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:20:15.783971Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:20:15.785044Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:20:16.131471Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:20:16.132214Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:20:16.476171Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.344s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.343910711 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:20:16.476276Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:20:16.476726Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:20:16.476739Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:20:16.476859Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:20:18.184508Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:20:18.184538Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:20:18.185014Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:20:18.186055Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:20:18.209401Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.936368 first_bonus_ms=23.268603 first_hidden_ms=0.00696 bind_reset_ms=0.08532799999999999 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:20:18.209584Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1731 +2026-09-21T01:20:18.209592Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1732.185174 burst_active_ms=1732.185174 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:20:18.209642Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:20:18.209736Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:20:18.210587Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:20:18.212201Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:20:18.212208Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:20:18.212212Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:20:18.212213Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:20:18.212214Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:20:18.212216Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:20:18.212217Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:20:18.212218Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:20:18.212220Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:20:18.212221Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:20:18.212222Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:20:18.212223Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:20:18.212225Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:20:24.656830Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=4.958911 first_bonus_ms=581.384978 first_hidden_ms=0.0076159999999999995 bind_reset_ms=0.027455999999999998 draft_ms=2244.517595999999 verify_ms=178.152222 target_argmax_sync_ms=2492.190311999999 logprobs_ms=0.008688000000000001 walk_ms=0.06519999999999998 hidden_concat_ms=0.13099200000000003 rollback_ms=43.242294000000015 decode_ms=4967.833798 +2026-09-21T01:20:24.657424Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5555 +2026-09-21T01:20:24.657465Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5555.019415000001 burst_active_ms=5555.019415000001 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:20:27.649613Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.808367 first_bonus_ms=120.03227 first_hidden_ms=0.015216 bind_reset_ms=0.086592 draft_ms=269.96634900000004 verify_ms=183.84626500000002 target_argmax_sync_ms=2360.524236 logprobs_ms=0.006976000000000007 walk_ms=0.06059199999999999 hidden_concat_ms=0.12984 rollback_ms=41.285243999999985 decode_ms=2864.3033889999997 +2026-09-21T01:20:27.649944Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=2987 +2026-09-21T01:20:27.649990Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=2987.732986 burst_active_ms=2987.732986 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.A.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.A.wclassic.log new file mode 100644 index 000000000..caafcbe88 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.A.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:23:55.633648Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:23:55.633687Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:23:55.633821Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:23:55.633856Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:23:55.633859Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:23:55.633861Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:23:55.956984Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:23:55.957021Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:23:55.957565Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:23:56.282903Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:23:56.283418Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:23:56.640992Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.358s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.357537476 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:23:56.641065Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:23:56.641466Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:23:56.641480Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:23:56.641503Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:23:56.642682Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:23:57.561817Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 920ms prompt_tokens=1 cached_tokens=0 generation_time_ms=920 +2026-09-21T01:23:57.562158Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:23:57.562325Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:23:57.563711Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:23:57.564504Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:23:57.564522Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:23:57.564525Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:23:57.564527Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:23:57.564528Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:23:57.564529Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:23:57.564531Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:23:57.564532Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:23:57.564533Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:23:57.564534Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:23:57.564536Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:23:57.564537Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:23:57.564538Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:24:02.251449Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3956ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3956 +2026-09-21T01:24:05.717952Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3464ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3464 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.B.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.B.w4.log new file mode 100644 index 000000000..11287edec --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.B.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:21:28.972121Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:21:28.972204Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:21:28.972682Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:21:28.972832Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:21:28.972834Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:21:28.972837Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:21:29.315742Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:21:29.315872Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:21:29.317076Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:21:29.667089Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:21:29.667535Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:21:30.000496Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.333s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.332941366 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:21:30.000610Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:21:30.001086Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:21:30.001101Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:21:30.001254Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:21:31.661127Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:21:31.661162Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:21:31.661638Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:21:31.662670Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:21:31.685081Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.959296 first_bonus_ms=22.345471 first_hidden_ms=0.00832 bind_reset_ms=0.06572800000000001 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:21:31.685266Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1683 +2026-09-21T01:21:31.685273Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1683.487288 burst_active_ms=1683.487288 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:21:31.685363Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:21:31.685464Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:21:31.686569Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:21:31.688294Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:21:31.688301Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:21:31.688304Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121a-real;121] (cubin) +2026-09-21T01:21:31.688306Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:21:31.688307Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:21:31.688308Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:21:31.688310Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:21:31.688311Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:21:31.688312Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:21:31.688313Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:21:31.688314Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:21:31.688316Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:21:31.688317Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:21:38.012816Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=4.938996 first_bonus_ms=579.548156 first_hidden_ms=0.010736 bind_reset_ms=0.03456 draft_ms=2117.74567 verify_ms=184.213324 target_argmax_sync_ms=2437.9943560000015 logprobs_ms=0.008671999999999996 walk_ms=0.052912 hidden_concat_ms=0.12183999999999999 rollback_ms=40.39651200000001 decode_ms=4788.528716 +2026-09-21T01:21:38.013404Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5373 +2026-09-21T01:21:38.013446Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5373.854765 burst_active_ms=5373.854765 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:21:40.970158Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.809027 first_bonus_ms=118.840216 first_hidden_ms=0.009008 bind_reset_ms=0.050831999999999995 draft_ms=246.11965800000002 verify_ms=183.91523999999998 target_argmax_sync_ms=2350.798859000001 logprobs_ms=0.005520000000000001 walk_ms=0.04375999999999998 hidden_concat_ms=0.125024 rollback_ms=41.34867499999999 decode_ms=2829.950116 +2026-09-21T01:21:40.970484Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=2952 +2026-09-21T01:21:40.970531Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=2952.15 burst_active_ms=2952.15 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.B.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.B.wclassic.log new file mode 100644 index 000000000..55df4e6d0 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.B.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:25:06.962802Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:25:06.962843Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:25:06.962984Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:25:06.963023Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:25:06.963026Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:25:06.963028Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:25:07.285000Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:25:07.285056Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:25:07.285606Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:25:07.611194Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:25:07.611917Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:25:07.982834Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.371s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.370860324 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:25:07.982913Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:25:07.983339Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:25:07.983354Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:25:07.983378Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:25:07.984561Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:25:08.911137Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 927ms prompt_tokens=1 cached_tokens=0 generation_time_ms=927 +2026-09-21T01:25:08.911481Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:25:08.911591Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:25:08.913237Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:25:08.914033Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:25:08.914050Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:25:08.914053Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121a-real;121] (cubin) +2026-09-21T01:25:08.914055Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:25:08.914056Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:25:08.914057Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:25:08.914058Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:25:08.914060Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:25:08.914061Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:25:08.914062Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:25:08.914063Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:25:08.914065Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:25:08.914066Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:25:13.646471Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 4010ms prompt_tokens=158 cached_tokens=0 generation_time_ms=4010 +2026-09-21T01:25:17.188886Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3539ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3539 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.C.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.C.w4.log new file mode 100644 index 000000000..190b30b08 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.C.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:22:42.306008Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:22:42.306081Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:22:42.306519Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:22:42.306605Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:22:42.306607Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:22:42.306609Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:22:42.652173Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:22:42.652302Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:22:42.653325Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:22:43.001089Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:22:43.001789Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:22:43.347044Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.345s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.345225008 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:22:43.347181Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:22:43.347692Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:22:43.347705Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:22:43.347850Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:22:44.966612Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:22:44.966642Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:22:44.967074Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:22:44.968112Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:22:44.991352Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.934642 first_bonus_ms=23.147521 first_hidden_ms=0.007632000000000001 bind_reset_ms=0.09614400000000001 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:22:44.991583Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1642 +2026-09-21T01:22:44.991590Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1643.2773140000002 burst_active_ms=1643.2773140000002 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:22:44.991687Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:22:44.991840Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:22:44.993827Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:22:44.995449Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:22:44.995456Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:22:44.995459Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:22:44.995460Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:22:44.995462Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:22:44.995463Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:22:44.995464Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:22:44.995465Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:22:44.995466Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:22:44.995468Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:22:44.995469Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:22:44.995470Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:22:44.995472Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:22:51.311629Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.014855000000001 first_bonus_ms=587.9365730000001 first_hidden_ms=0.010976 bind_reset_ms=0.036048 draft_ms=2052.1914949999996 verify_ms=185.70774500000002 target_argmax_sync_ms=2473.857789 logprobs_ms=0.007680000000000004 walk_ms=0.06049600000000001 hidden_concat_ms=0.13017800000000002 rollback_ms=42.41391000000001 decode_ms=4762.442584 +2026-09-21T01:22:51.311940Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5355 +2026-09-21T01:22:51.311983Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5355.928717000001 burst_active_ms=5355.928717000001 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:22:54.315935Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.869636 first_bonus_ms=118.403987 first_hidden_ms=0.01568 bind_reset_ms=0.087296 draft_ms=267.97565699999996 verify_ms=183.81967099999994 target_argmax_sync_ms=2376.9887470000003 logprobs_ms=0.007024000000000001 walk_ms=0.04948800000000001 hidden_concat_ms=0.127217 rollback_ms=42.65962700000001 decode_ms=2878.692496 +2026-09-21T01:22:54.316239Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=3000 +2026-09-21T01:22:54.316282Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=3000.541513 burst_active_ms=3000.541513 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.C.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.C.wclassic.log new file mode 100644 index 000000000..3d412ee34 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r1.C.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:26:18.427602Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:26:18.427648Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:26:18.427777Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:26:18.427831Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:26:18.427834Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:26:18.427836Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:26:18.773103Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:26:18.773142Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:26:18.773726Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:26:19.122234Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:26:19.122587Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:26:19.470490Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.348s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.347878309 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:26:19.470568Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:26:19.470976Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:26:19.470991Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:26:19.471012Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:26:19.472152Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:26:20.316073Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 844ms prompt_tokens=1 cached_tokens=0 generation_time_ms=844 +2026-09-21T01:26:20.316500Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:26:20.316596Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:26:20.318004Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:26:20.318793Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:26:20.318810Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:26:20.318813Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:26:20.318814Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:26:20.318816Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:26:20.318817Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:26:20.318818Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:26:20.318819Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:26:20.318821Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:26:20.318822Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:26:20.318823Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:26:20.318824Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:26:20.318826Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:26:25.073448Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3990ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3990 +2026-09-21T01:26:28.612306Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3536ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3536 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.A.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.A.w4.log new file mode 100644 index 000000000..5099d3b98 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.A.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:29:56.713839Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:29:56.713881Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:29:56.714020Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:29:56.714111Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:29:56.714113Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:29:56.714115Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:29:57.038378Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:29:57.038412Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:29:57.038992Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:29:57.366638Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:29:57.367044Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:29:57.734664Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.368s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.367546781 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:29:57.734741Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:29:57.735162Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:29:57.735178Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:29:57.735200Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:29:59.449353Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:29:59.449383Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:29:59.449793Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:29:59.450800Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:29:59.473865Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.962132 first_bonus_ms=23.027092 first_hidden_ms=0.009184000000000001 bind_reset_ms=0.038609000000000004 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:29:59.474033Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1738 +2026-09-21T01:29:59.474041Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1738.601338 burst_active_ms=1738.601338 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:29:59.474777Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:29:59.474882Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:29:59.475731Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:29:59.476531Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:29:59.476538Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:29:59.476541Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:29:59.476543Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:29:59.476544Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:29:59.476545Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:29:59.476546Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:29:59.476548Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:29:59.476549Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:29:59.476550Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:29:59.476551Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:29:59.476553Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:29:59.476554Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:30:05.761019Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.0182459999999995 first_bonus_ms=595.284883 first_hidden_ms=0.009104 bind_reset_ms=0.031296 draft_ms=2125.874181999999 verify_ms=175.27199600000003 target_argmax_sync_ms=2433.895492 logprobs_ms=0.012832000000000005 walk_ms=0.057567999999999994 hidden_concat_ms=0.128049 rollback_ms=43.016557000000006 decode_ms=4785.179344 +2026-09-21T01:30:05.761315Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5385 +2026-09-21T01:30:05.761352Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5385.988474000001 burst_active_ms=5385.988474000001 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:30:08.768331Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.726636 first_bonus_ms=117.09523300000001 first_hidden_ms=0.0072 bind_reset_ms=0.046656 draft_ms=272.8918859999999 verify_ms=178.38529700000004 target_argmax_sync_ms=2382.083207 logprobs_ms=0.009984 walk_ms=0.044464 hidden_concat_ms=0.13188800000000003 rollback_ms=43.719018999999996 decode_ms=2883.853979 +2026-09-21T01:30:08.768660Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=3004 +2026-09-21T01:30:08.768708Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=3004.254522 burst_active_ms=3004.254522 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.A.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.A.wclassic.log new file mode 100644 index 000000000..dfdf83c6c --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.A.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:33:33.171547Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:33:33.171592Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:33:33.171727Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:33:33.171763Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:33:33.171765Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:33:33.171767Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:33:33.495732Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:33:33.495765Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:33:33.496319Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:33:33.821105Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:33:33.821756Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:33:34.178396Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.357s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.356603466 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:33:34.178468Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:33:34.178884Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:33:34.178898Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:33:34.178921Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:33:34.180103Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:33:35.137615Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 958ms prompt_tokens=1 cached_tokens=0 generation_time_ms=958 +2026-09-21T01:33:35.138603Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:33:35.138705Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:33:35.140085Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:33:35.140854Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:33:35.140870Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:33:35.140873Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:33:35.140875Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:33:35.140876Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:33:35.140877Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:33:35.140878Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:33:35.140880Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:33:35.140881Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:33:35.140882Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:33:35.140883Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:33:35.140885Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:33:35.140886Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:33:39.910139Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 4065ms prompt_tokens=158 cached_tokens=0 generation_time_ms=4065 +2026-09-21T01:33:43.500287Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3587ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3587 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.B.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.B.w4.log new file mode 100644 index 000000000..a0813822c --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.B.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:27:29.907560Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:27:29.907608Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:27:29.907745Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:27:29.907818Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:27:29.907820Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:27:29.907823Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:27:30.258217Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:27:30.258256Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:27:30.258844Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:27:30.619036Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:27:30.619525Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:27:30.951983Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.332s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.332412932 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:27:30.952059Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:27:30.952455Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:27:30.952469Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:27:30.952494Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:27:32.657219Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:27:32.657248Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:27:32.657642Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:27:32.658604Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:27:32.681685Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.9187559999999999 first_bonus_ms=23.061315 first_hidden_ms=0.007025 bind_reset_ms=0.02576 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:27:32.681833Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1728 +2026-09-21T01:27:32.681840Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1729.13749 burst_active_ms=1729.13749 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:27:32.682038Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:27:32.682201Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:27:32.683099Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:27:32.683871Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:27:32.683877Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:27:32.683881Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121a-real;121] (cubin) +2026-09-21T01:27:32.683882Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:27:32.683884Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:27:32.683885Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:27:32.683886Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:27:32.683887Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:27:32.683889Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:27:32.683890Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:27:32.683891Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:27:32.683892Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:27:32.683894Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:27:38.955426Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.022322 first_bonus_ms=581.439812 first_hidden_ms=0.011151999999999999 bind_reset_ms=0.03528 draft_ms=2123.2643770000013 verify_ms=176.516361 target_argmax_sync_ms=2434.534705 logprobs_ms=0.009552000000000001 walk_ms=0.06516799999999998 hidden_concat_ms=0.12198399999999998 rollback_ms=46.07206899999999 decode_ms=4789.819776 +2026-09-21T01:27:38.955753Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5376 +2026-09-21T01:27:38.955794Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5376.9176720000005 burst_active_ms=5376.9176720000005 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:27:41.889840Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.960907 first_bonus_ms=117.103213 first_hidden_ms=0.014735999999999999 bind_reset_ms=0.082385 draft_ms=258.337034 verify_ms=184.72328 target_argmax_sync_ms=2317.547057999999 logprobs_ms=0.009089000000000002 walk_ms=0.049936000000000015 hidden_concat_ms=0.146768 rollback_ms=43.11251200000001 decode_ms=2809.8846970000004 +2026-09-21T01:27:41.890179Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=2930 +2026-09-21T01:27:41.890226Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=2930.5457170000004 burst_active_ms=2930.5457170000004 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.B.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.B.wclassic.log new file mode 100644 index 000000000..8d56dae47 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.B.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:31:10.055176Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:31:10.055223Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:31:10.055365Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:31:10.055448Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:31:10.055450Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:31:10.055453Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:31:10.398646Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:31:10.398681Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:31:10.399258Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:31:10.745725Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:31:10.746211Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:31:11.081095Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.335s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.334851408 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:31:11.081170Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:31:11.081566Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:31:11.081579Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:31:11.081603Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:31:11.082726Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:31:11.970445Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 888ms prompt_tokens=1 cached_tokens=0 generation_time_ms=888 +2026-09-21T01:31:11.970510Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:31:11.970600Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:31:11.971264Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:31:11.972069Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:31:11.972087Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:31:11.972090Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121a-real;121] (cubin) +2026-09-21T01:31:11.972092Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:31:11.972093Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:31:11.972094Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:31:11.972096Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:31:11.972097Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:31:11.972098Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:31:11.972099Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:31:11.972101Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:31:11.972102Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:31:11.972103Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:31:16.729079Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 4016ms prompt_tokens=158 cached_tokens=0 generation_time_ms=4016 +2026-09-21T01:31:20.304633Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3573ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3573 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.C.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.C.w4.log new file mode 100644 index 000000000..c5edc66f0 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.C.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:28:43.174765Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:28:43.174804Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:28:43.174937Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:28:43.174982Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:28:43.174984Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:28:43.174986Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:28:43.498068Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:28:43.498104Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:28:43.498646Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:28:43.822648Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:28:43.823170Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:28:44.185084Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.362s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.361842427 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:28:44.185158Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:28:44.185576Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:28:44.185590Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:28:44.185612Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:28:45.918767Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:28:45.918798Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:28:45.919212Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:28:45.920240Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:28:45.943564Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.9829960000000001 first_bonus_ms=23.298060999999997 first_hidden_ms=0.008 bind_reset_ms=0.030272 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:28:45.943737Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1757 +2026-09-21T01:28:45.943745Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1757.9108159999998 burst_active_ms=1757.9108159999998 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:28:45.943879Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:28:45.943970Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:28:45.945238Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:28:45.946007Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:28:45.946017Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:28:45.946020Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:28:45.946022Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:28:45.946023Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:28:45.946024Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:28:45.946026Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:28:45.946027Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:28:45.946028Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:28:45.946029Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:28:45.946031Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:28:45.946032Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:28:45.946033Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:28:52.366952Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.042245 first_bonus_ms=599.599293 first_hidden_ms=0.0112 bind_reset_ms=0.034048 draft_ms=2160.3745980000003 verify_ms=180.86351299999998 target_argmax_sync_ms=2516.9336369999996 logprobs_ms=0.010752000000000001 walk_ms=0.08198500000000002 hidden_concat_ms=0.135905 rollback_ms=43.608985000000004 decode_ms=4909.299134 +2026-09-21T01:28:52.367526Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5514 +2026-09-21T01:28:52.367566Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5514.730834 burst_active_ms=5514.730834 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:28:55.425337Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.7640439999999997 first_bonus_ms=124.301591 first_hidden_ms=0.009616000000000001 bind_reset_ms=0.058096 draft_ms=276.4062280000001 verify_ms=173.241899 target_argmax_sync_ms=2427.1663460000004 logprobs_ms=0.008623999999999996 walk_ms=0.06824000000000001 hidden_concat_ms=0.128368 rollback_ms=42.880556000000006 decode_ms=2926.59653 +2026-09-21T01:28:55.425682Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=3054 +2026-09-21T01:28:55.425733Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=3054.237815 burst_active_ms=3054.237815 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.C.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.C.wclassic.log new file mode 100644 index 000000000..4248c8551 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r2.C.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:32:21.618456Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:32:21.618502Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:32:21.618632Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:32:21.618710Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:32:21.618713Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:32:21.618715Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:32:21.962538Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:32:21.962576Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:32:21.963139Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:32:22.314150Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:32:22.314965Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:32:22.649242Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.334s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.334238048 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:32:22.649310Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:32:22.649713Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:32:22.649726Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:32:22.649744Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:32:22.650893Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:32:23.526856Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 876ms prompt_tokens=1 cached_tokens=0 generation_time_ms=876 +2026-09-21T01:32:23.526911Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:32:23.527016Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:32:23.527985Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:32:23.528753Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:32:23.528773Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:32:23.528776Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:32:23.528778Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:32:23.528779Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:32:23.528780Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:32:23.528781Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:32:23.528783Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:32:23.528784Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:32:23.528785Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:32:23.528786Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:32:23.528788Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:32:23.528789Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:32:28.291777Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 4022ms prompt_tokens=158 cached_tokens=0 generation_time_ms=4022 +2026-09-21T01:32:31.922218Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3628ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3628 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.A.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.A.w4.log new file mode 100644 index 000000000..524df1cc5 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.A.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:35:58.051002Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:35:58.051049Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:35:58.051183Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:35:58.051254Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:35:58.051256Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:35:58.051258Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:35:58.396493Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:35:58.396528Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:35:58.397105Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:35:58.749112Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:35:58.749579Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:35:59.084655Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.335s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.335033073 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:35:59.084735Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:35:59.085136Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:35:59.085152Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:35:59.085176Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:36:00.726637Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:36:00.726668Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:36:00.727070Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:36:00.728028Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:36:00.751699Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.9150929999999999 first_bonus_ms=23.626689 first_hidden_ms=0.009904 bind_reset_ms=0.043664 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:36:00.751875Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1666 +2026-09-21T01:36:00.751884Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1666.499419 burst_active_ms=1666.499419 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:36:00.751885Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:36:00.751982Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:36:00.753076Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:36:00.753791Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:36:00.753796Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:36:00.753800Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:36:00.753801Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:36:00.753803Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:36:00.753804Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:36:00.753805Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:36:00.753807Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:36:00.753808Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:36:00.753809Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:36:00.753810Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:36:00.753812Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:36:00.753813Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:36:07.069642Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=4.943932 first_bonus_ms=584.0123860000001 first_hidden_ms=0.019232000000000003 bind_reset_ms=0.05968 draft_ms=2060.6090190000004 verify_ms=178.89819000000003 target_argmax_sync_ms=2482.075442 logprobs_ms=0.010064000000000005 walk_ms=0.06916800000000001 hidden_concat_ms=0.125136 rollback_ms=43.88679900000001 decode_ms=4773.435041 +2026-09-21T01:36:07.070174Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5363 +2026-09-21T01:36:07.070215Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5363.179394000001 burst_active_ms=5363.179394000001 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:36:10.102567Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.783391 first_bonus_ms=119.717247 first_hidden_ms=0.008688000000000001 bind_reset_ms=0.048288000000000005 draft_ms=268.06839699999995 verify_ms=189.78366400000002 target_argmax_sync_ms=2392.2849709999996 logprobs_ms=0.007008000000000003 walk_ms=0.057376000000000024 hidden_concat_ms=0.140913 rollback_ms=47.032144 decode_ms=2905.187327 +2026-09-21T01:36:10.102887Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=3028 +2026-09-21T01:36:10.102935Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=3028.242095 burst_active_ms=3028.242095 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.A.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.A.wclassic.log new file mode 100644 index 000000000..302677617 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.A.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:39:36.252176Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:39:36.252217Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:39:36.252350Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:39:36.252386Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:39:36.252388Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:39:36.252390Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:39:36.573783Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:39:36.573819Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:39:36.574389Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:39:36.902654Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:39:36.902831Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:39:37.276007Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.373s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.373150204 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:39:37.276086Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:39:37.276495Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:39:37.276508Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:39:37.276533Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:39:37.277704Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:39:38.205615Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 928ms prompt_tokens=1 cached_tokens=0 generation_time_ms=928 +2026-09-21T01:39:38.205709Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:39:38.205797Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:39:38.206401Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:39:38.207187Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:39:38.207205Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:39:38.207208Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:39:38.207209Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:39:38.207211Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:39:38.207212Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:39:38.207213Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:39:38.207214Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:39:38.207216Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:39:38.207217Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:39:38.207218Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:39:38.207219Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:39:38.207221Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:39:42.905402Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3982ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3982 +2026-09-21T01:39:46.394062Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3486ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3486 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.B.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.B.w4.log new file mode 100644 index 000000000..e110b90ac --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.B.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:37:11.471581Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:37:11.471621Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:37:11.471756Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:37:11.471793Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:37:11.471796Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:37:11.471798Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:37:11.794944Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:37:11.794979Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:37:11.795527Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:37:12.124857Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:37:12.125617Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:37:12.493640Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.368s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.367956137 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:37:12.493722Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:37:12.494155Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:37:12.494171Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:37:12.494199Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:37:14.168003Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:37:14.168032Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:37:14.168437Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:37:14.169400Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:37:14.193412Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.919381 first_bonus_ms=23.982502 first_hidden_ms=0.007712 bind_reset_ms=0.030255999999999998 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:37:14.193590Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1698 +2026-09-21T01:37:14.193598Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1699.1681600000002 burst_active_ms=1699.1681600000002 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:37:14.193813Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:37:14.193981Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:37:14.195115Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:37:14.195940Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:37:14.195947Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:37:14.195951Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121a-real;121] (cubin) +2026-09-21T01:37:14.195952Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:37:14.195954Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:37:14.195955Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:37:14.195956Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:37:14.195957Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:37:14.195959Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:37:14.195960Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:37:14.195961Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:37:14.195962Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:37:14.195964Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:37:20.534788Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.050556 first_bonus_ms=595.9546959999999 first_hidden_ms=0.010737 bind_reset_ms=0.035775999999999995 draft_ms=2124.335785999999 verify_ms=183.52257600000002 target_argmax_sync_ms=2436.8887019999997 logprobs_ms=0.017936000000000004 walk_ms=0.06359999999999998 hidden_concat_ms=0.12744099999999997 rollback_ms=43.556308999999985 decode_ms=4795.475762 +2026-09-21T01:37:20.535372Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5397 +2026-09-21T01:37:20.535414Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5397.336956 burst_active_ms=5397.336956 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:37:23.498764Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.901024 first_bonus_ms=119.69748700000001 first_hidden_ms=0.015776 bind_reset_ms=0.090817 draft_ms=266.3839079999999 verify_ms=182.06360999999995 target_argmax_sync_ms=2338.1235519999996 logprobs_ms=0.015007999999999999 walk_ms=0.052033 hidden_concat_ms=0.135936 rollback_ms=43.39360400000001 decode_ms=2835.84458 +2026-09-21T01:37:23.499130Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=2959 +2026-09-21T01:37:23.499181Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=2959.077352 burst_active_ms=2959.077352 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.B.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.B.wclassic.log new file mode 100644 index 000000000..986f1ebb1 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.B.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:40:47.637010Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:40:47.637055Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:40:47.637192Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:40:47.637292Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:40:47.637294Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:40:47.637296Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:40:47.963141Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:40:47.963175Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:40:47.963740Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:40:48.290799Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:40:48.290803Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:40:48.649606Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.359s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.358767721 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:40:48.649681Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:40:48.650110Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:40:48.650125Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:40:48.650150Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:40:48.651342Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:40:49.579834Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 929ms prompt_tokens=1 cached_tokens=0 generation_time_ms=929 +2026-09-21T01:40:49.580007Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:40:49.580153Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:40:49.581467Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:40:49.582249Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:40:49.582265Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:40:49.582269Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121a-real;121] (cubin) +2026-09-21T01:40:49.582270Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:40:49.582272Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:40:49.582273Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:40:49.582274Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:40:49.582275Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:40:49.582277Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:40:49.582278Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:40:49.582279Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:40:49.582280Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:40:49.582282Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:40:54.256569Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3962ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3962 +2026-09-21T01:40:57.751219Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3492ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3492 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.C.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.C.w4.log new file mode 100644 index 000000000..025be6d8b --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.C.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:34:44.747736Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:34:44.747778Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:34:44.747907Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:34:44.747998Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:34:44.748002Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:34:44.748004Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:34:45.091247Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:34:45.091283Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:34:45.091854Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:34:45.446224Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:34:45.446669Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:34:45.781623Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.335s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.334933353 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:34:45.781693Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:34:45.782092Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:34:45.782107Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:34:45.782126Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:34:47.396271Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:34:47.396301Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:34:47.396709Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:34:47.397719Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:34:47.420129Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.9668209999999999 first_bonus_ms=22.379335 first_hidden_ms=0.00832 bind_reset_ms=0.033472 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:34:47.420285Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1637 +2026-09-21T01:34:47.420292Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1637.9603849999999 burst_active_ms=1637.9603849999999 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:34:47.420360Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:34:47.420445Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:34:47.420959Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:34:47.421723Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:34:47.421729Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:34:47.421732Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:34:47.421734Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:34:47.421735Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:34:47.421737Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:34:47.421738Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:34:47.421740Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:34:47.421741Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:34:47.421743Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:34:47.421745Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:34:47.421746Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:34:47.421748Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:34:53.743331Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.092971 first_bonus_ms=582.220952 first_hidden_ms=0.008864 bind_reset_ms=0.028928000000000002 draft_ms=2057.86943 verify_ms=178.767745 target_argmax_sync_ms=2440.3703560000004 logprobs_ms=0.008144000000000005 walk_ms=0.07014600000000001 hidden_concat_ms=0.12454399999999997 rollback_ms=45.42827300000002 decode_ms=4731.529863000001 +2026-09-21T01:34:53.743888Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5319 +2026-09-21T01:34:53.743936Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5319.6241899999995 burst_active_ms=5319.6241899999995 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:34:56.721598Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.9102870000000003 first_bonus_ms=121.29859499999999 first_hidden_ms=0.015888 bind_reset_ms=0.075489 draft_ms=254.99693900000008 verify_ms=183.19217499999993 target_argmax_sync_ms=2356.811484 logprobs_ms=0.007712000000000004 walk_ms=0.062496 hidden_concat_ms=0.11846400000000001 rollback_ms=45.15577400000001 decode_ms=2848.891869 +2026-09-21T01:34:56.721901Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=2973 +2026-09-21T01:34:56.721952Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=2973.738659 burst_active_ms=2973.738659 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.C.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.C.wclassic.log new file mode 100644 index 000000000..b259281f2 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r3.C.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:38:24.827424Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:38:24.827463Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:38:24.827588Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:38:24.827626Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:38:24.827628Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:38:24.827630Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:38:25.151895Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:38:25.151937Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:38:25.152483Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:38:25.480709Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:38:25.480720Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:38:25.837237Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.356s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.356494422 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:38:25.837314Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:38:25.837720Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:38:25.837734Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:38:25.837754Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:38:25.838919Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:38:26.776183Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 938ms prompt_tokens=1 cached_tokens=0 generation_time_ms=938 +2026-09-21T01:38:26.776426Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:38:26.776524Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:38:26.777731Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:38:26.778512Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:38:26.778529Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:38:26.778533Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:38:26.778534Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:38:26.778535Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:38:26.778537Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:38:26.778538Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:38:26.778539Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:38:26.778541Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:38:26.778542Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:38:26.778543Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:38:26.778545Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:38:26.778546Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:38:31.466951Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3982ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3982 +2026-09-21T01:38:34.973956Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3505ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3505 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.A.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.A.w4.log new file mode 100644 index 000000000..e15e01e18 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.A.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:41:58.994461Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:41:58.994508Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:41:58.994640Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:41:58.994729Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:41:58.994731Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:41:58.994733Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:41:59.318940Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:41:59.318976Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:41:59.319520Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:41:59.645560Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:41:59.646107Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:42:00.014391Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.368s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.368241374 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:42:00.014468Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:42:00.014878Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:42:00.014891Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:42:00.014914Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:42:01.693209Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:42:01.693239Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:42:01.693642Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:42:01.694657Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:42:01.717612Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.970706 first_bonus_ms=22.9166 first_hidden_ms=0.009311999999999999 bind_reset_ms=0.039230999999999995 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:42:01.717777Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1702 +2026-09-21T01:42:01.717784Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1702.6441810000001 burst_active_ms=1702.6441810000001 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:42:01.718500Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:42:01.718603Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:42:01.719437Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:42:01.720222Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:42:01.720230Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:42:01.720233Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:42:01.720234Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:42:01.720236Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:42:01.720237Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:42:01.720238Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:42:01.720239Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:42:01.720241Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:42:01.720242Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:42:01.720243Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:42:01.720244Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:42:01.720246Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:42:08.037491Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.0690789999999994 first_bonus_ms=593.5414380000001 first_hidden_ms=0.008816 bind_reset_ms=0.028815 draft_ms=2118.4096140000006 verify_ms=175.59244000000004 target_argmax_sync_ms=2433.411687 logprobs_ms=0.011598000000000002 walk_ms=0.054143000000000024 hidden_concat_ms=0.11914900000000002 rollback_ms=39.214659 decode_ms=4774.0627 +2026-09-21T01:42:08.038117Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5373 +2026-09-21T01:42:08.038170Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5373.5134450000005 burst_active_ms=5373.5134450000005 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:42:10.986475Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.7498810000000002 first_bonus_ms=119.478464 first_hidden_ms=0.008304 bind_reset_ms=0.046255 draft_ms=259.74100899999996 verify_ms=179.4162 target_argmax_sync_ms=2337.9900299999995 logprobs_ms=0.010384000000000001 walk_ms=0.04655999999999999 hidden_concat_ms=0.12988499999999992 rollback_ms=39.725601000000005 decode_ms=2822.461017 +2026-09-21T01:42:10.986802Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=2945 +2026-09-21T01:42:10.986848Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=2945.209194 burst_active_ms=2945.209194 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.A.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.A.wclassic.log new file mode 100644 index 000000000..66866597d --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.A.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:45:39.036123Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:45:39.036166Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:45:39.036296Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:45:39.036332Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:45:39.036335Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:45:39.036337Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:45:39.361268Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:45:39.361304Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:45:39.361858Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:45:39.684741Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:45:39.685487Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:45:40.052724Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.367s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.367161552 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:45:40.052797Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:45:40.053219Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:45:40.053235Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:45:40.053259Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:45:40.054422Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:45:40.987843Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 934ms prompt_tokens=1 cached_tokens=0 generation_time_ms=934 +2026-09-21T01:45:40.987916Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:45:40.988016Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:45:40.989010Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:45:40.989788Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:45:40.989804Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:45:40.989807Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:45:40.989809Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:45:40.989810Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:45:40.989811Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:45:40.989813Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:45:40.989814Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:45:40.989815Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:45:40.989816Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:45:40.989818Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:45:40.989819Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:45:40.989820Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:45:45.736627Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 4041ms prompt_tokens=158 cached_tokens=0 generation_time_ms=4041 +2026-09-21T01:45:49.315000Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3576ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3576 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.B.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.B.w4.log new file mode 100644 index 000000000..2b62cef9d --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.B.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:43:12.283290Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:43:12.283331Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:43:12.283467Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:43:12.283503Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:43:12.283505Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:43:12.283508Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:43:12.608805Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:43:12.608840Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:43:12.609402Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:43:12.935789Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:43:12.936356Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:43:13.302732Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.366s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.366304973 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:43:13.302816Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:43:13.303245Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:43:13.303262Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:43:13.303287Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:43:14.954222Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:43:14.954257Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:43:14.954658Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:43:14.955654Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:43:14.978875Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.952788 first_bonus_ms=23.175806 first_hidden_ms=0.009999000000000001 bind_reset_ms=0.040528 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:43:14.979051Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1675 +2026-09-21T01:43:14.979058Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1675.544804 burst_active_ms=1675.544804 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:43:14.979241Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:43:14.979333Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:43:14.980593Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:43:14.981365Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:43:14.981372Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:43:14.981375Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121a-real;121] (cubin) +2026-09-21T01:43:14.981377Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:43:14.981378Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:43:14.981379Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:43:14.981380Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:43:14.981382Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:43:14.981383Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:43:14.981384Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:43:14.981385Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:43:14.981387Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:43:14.981388Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:43:21.356055Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.010725 first_bonus_ms=594.866041 first_hidden_ms=0.0108 bind_reset_ms=0.032783 draft_ms=2125.8520889999995 verify_ms=187.34871900000002 target_argmax_sync_ms=2438.0010700000003 logprobs_ms=0.011439999999999997 walk_ms=0.05662399999999999 hidden_concat_ms=0.13371099999999997 rollback_ms=43.378983 decode_ms=4802.527433 +2026-09-21T01:43:21.356609Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5403 +2026-09-21T01:43:21.356650Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5403.173262 burst_active_ms=5403.173262 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:43:24.332426Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.863135 first_bonus_ms=119.60056 first_hidden_ms=0.015856 bind_reset_ms=0.098703 draft_ms=258.218033 verify_ms=184.78606400000004 target_argmax_sync_ms=2357.277712000001 logprobs_ms=0.007664 walk_ms=0.04937599999999999 hidden_concat_ms=0.12775899999999996 rollback_ms=43.760643 decode_ms=2850.464603 +2026-09-21T01:43:24.332730Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=2973 +2026-09-21T01:43:24.332775Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=2973.498996 burst_active_ms=2973.498996 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.B.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.B.wclassic.log new file mode 100644 index 000000000..8f6d7e420 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.B.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:46:50.518453Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:46:50.518498Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:46:50.518633Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:46:50.518699Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:46:50.518702Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:46:50.518704Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:46:50.844301Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:46:50.844338Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:46:50.844885Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:46:51.170454Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:46:51.170969Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:46:51.533152Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.362s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.362138196 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:46:51.533229Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:46:51.533653Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:46:51.533667Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:46:51.533693Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:46:51.534891Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:46:52.503829Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 969ms prompt_tokens=1 cached_tokens=0 generation_time_ms=969 +2026-09-21T01:46:52.504655Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:46:52.504813Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:46:52.506091Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:46:52.506863Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:46:52.506879Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:46:52.506882Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121a-real;121] (cubin) +2026-09-21T01:46:52.506884Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:46:52.506885Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:46:52.506886Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:46:52.506887Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:46:52.506889Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:46:52.506890Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:46:52.506891Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:46:52.506893Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:46:52.506894Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:46:52.506895Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:46:57.162104Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3980ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3980 +2026-09-21T01:47:00.670413Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3505ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3505 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.C.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.C.w4.log new file mode 100644 index 000000000..0cedfb412 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.C.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:44:25.654259Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:44:25.654304Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:44:25.654436Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:44:25.654524Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:44:25.654527Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:44:25.654529Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:44:25.998263Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:44:25.998298Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:44:25.998866Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:44:26.349289Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:44:26.350071Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:44:26.696184Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.346s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.346073251 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:44:26.696258Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:44:26.696649Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:44:26.696663Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:44:26.696683Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:44:28.326397Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:44:28.326430Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:44:28.326818Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:44:28.327775Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:44:28.350454Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.915015 first_bonus_ms=22.632548 first_hidden_ms=0.010224 bind_reset_ms=0.043776 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:44:28.350618Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1653 +2026-09-21T01:44:28.350625Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1653.731 burst_active_ms=1653.731 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:44:28.350710Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:44:28.350805Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:44:28.352563Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:44:28.353334Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:44:28.353341Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:44:28.353345Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:44:28.353346Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:44:28.353348Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:44:28.353349Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:44:28.353350Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:44:28.353352Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:44:28.353353Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:44:28.353354Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:44:28.353356Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:44:28.353357Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:44:28.353359Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:44:34.670707Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.048863 first_bonus_ms=583.406906 first_hidden_ms=0.009648 bind_reset_ms=0.032014999999999995 draft_ms=2076.686953 verify_ms=177.15159699999998 target_argmax_sync_ms=2460.0480639999996 logprobs_ms=0.008528000000000003 walk_ms=0.05668800000000001 hidden_concat_ms=0.12689599999999995 rollback_ms=42.99402200000001 decode_ms=4765.233806 +2026-09-21T01:44:34.671277Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5354 +2026-09-21T01:44:34.671318Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5354.478110999999 burst_active_ms=5354.478110999999 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:44:37.689206Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.817829 first_bonus_ms=118.744885 first_hidden_ms=0.015328 bind_reset_ms=0.08291100000000001 draft_ms=251.316414 verify_ms=183.31045099999997 target_argmax_sync_ms=2405.8784010000013 logprobs_ms=0.007696000000000002 walk_ms=0.051037000000000006 hidden_concat_ms=0.114236 rollback_ms=44.101374 decode_ms=2891.754367 +2026-09-21T01:44:37.689537Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=3013 +2026-09-21T01:44:37.689580Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=3013.923667 burst_active_ms=3013.923667 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.C.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.C.wclassic.log new file mode 100644 index 000000000..9101ffd38 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r4.C.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:48:01.965601Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:48:01.965649Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:48:01.965778Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:48:01.965853Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:48:01.965855Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:48:01.965858Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:48:02.309730Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:48:02.309770Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:48:02.310338Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:48:02.656589Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:48:02.657096Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:48:02.994310Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.337s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.337161661 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:48:02.994380Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:48:02.994773Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:48:02.994786Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:48:02.994805Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:48:02.995936Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:48:03.878020Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 883ms prompt_tokens=1 cached_tokens=0 generation_time_ms=883 +2026-09-21T01:48:03.878177Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:48:03.878321Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:48:03.878985Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:48:03.879749Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:48:03.879764Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:48:03.879767Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:48:03.879769Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:48:03.879770Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:48:03.879772Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:48:03.879773Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:48:03.879774Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:48:03.879776Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:48:03.879777Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:48:03.879778Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:48:03.879779Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:48:03.879781Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:48:08.611744Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3979ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3979 +2026-09-21T01:48:12.193463Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3579ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3579 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.A.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.A.w4.log new file mode 100644 index 000000000..e19017649 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.A.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:51:40.254107Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:51:40.254147Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:51:40.254282Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:51:40.254318Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:51:40.254320Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:51:40.254322Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:51:40.577440Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:51:40.577478Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:51:40.578044Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:51:40.903599Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:51:40.903643Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:51:41.270603Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.367s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.3669274 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:51:41.270679Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:51:41.271090Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:51:41.271105Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:51:41.271130Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:51:42.978391Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:51:42.978421Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:51:42.978829Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:51:42.979861Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:51:43.002125Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.9877270000000001 first_bonus_ms=22.232937 first_hidden_ms=0.007872 bind_reset_ms=0.032688 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:51:43.002295Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1730 +2026-09-21T01:51:43.002303Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1730.95285 burst_active_ms=1730.95285 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:51:43.002461Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:51:43.002607Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:51:43.003233Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:51:43.004034Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:51:43.004040Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:51:43.004044Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:51:43.004045Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:51:43.004047Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:51:43.004048Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:51:43.004049Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:51:43.004050Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:51:43.004052Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:51:43.004053Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:51:43.004054Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:51:43.004056Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:51:43.004057Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:51:49.294377Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.078167 first_bonus_ms=593.435695 first_hidden_ms=0.010912 bind_reset_ms=0.031984 draft_ms=2119.0758200000005 verify_ms=190.83996199999999 target_argmax_sync_ms=2426.652814 logprobs_ms=0.012432000000000004 walk_ms=0.06550399999999999 hidden_concat_ms=0.14171200000000006 rollback_ms=43.309204 decode_ms=4787.517066 +2026-09-21T01:51:49.294966Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5386 +2026-09-21T01:51:49.295025Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5386.834239 burst_active_ms=5386.834239 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:51:52.248429Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.8956429999999997 first_bonus_ms=118.900649 first_hidden_ms=0.00816 bind_reset_ms=0.047072 draft_ms=261.52825899999993 verify_ms=182.45939100000007 target_argmax_sync_ms=2333.550974 logprobs_ms=0.009488 walk_ms=0.049999999999999996 hidden_concat_ms=0.13895900000000003 rollback_ms=43.22134900000002 decode_ms=2827.54142 +2026-09-21T01:51:52.248747Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=2949 +2026-09-21T01:51:52.248792Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=2949.8579029999996 burst_active_ms=2949.8579029999996 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.A.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.A.wclassic.log new file mode 100644 index 000000000..07841faa6 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.A.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:55:16.304225Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:55:16.304264Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:55:16.304395Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:55:16.304449Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:55:16.304451Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:55:16.304453Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:55:16.629312Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:55:16.629350Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:55:16.629915Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:55:16.963066Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:55:16.963432Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:55:17.298020Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.335s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.334568736 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:55:17.298092Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:55:17.298486Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:55:17.298499Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:55:17.298520Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:55:17.299646Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:55:18.183271Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 884ms prompt_tokens=1 cached_tokens=0 generation_time_ms=884 +2026-09-21T01:55:18.183337Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:55:18.183426Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:55:18.184067Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:55:18.184850Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:55:18.184867Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:55:18.184870Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:55:18.184871Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:55:18.184873Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:55:18.184874Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:55:18.184875Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:55:18.184877Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:55:18.184878Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:55:18.184879Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:55:18.184880Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:55:18.184881Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:55:18.184883Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:55:22.861784Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3877ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3877 +2026-09-21T01:55:26.229456Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3366ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3366 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.B.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.B.w4.log new file mode 100644 index 000000000..3c7812fb2 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.B.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:49:13.527958Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:49:13.528003Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:49:13.528141Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:49:13.528234Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:49:13.528236Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:49:13.528238Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:49:13.872030Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:49:13.872062Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:49:13.872626Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:49:14.222121Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:49:14.222135Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:49:14.557502Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.335s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.335348232 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:49:14.557578Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:49:14.557989Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:49:14.558004Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:49:14.558027Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:49:16.150390Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:49:16.150418Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:49:16.150826Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:49:16.151810Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:49:16.174371Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.94126 first_bonus_ms=22.538764 first_hidden_ms=0.007248 bind_reset_ms=0.026336 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:49:16.174537Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1615 +2026-09-21T01:49:16.174543Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1616.292896 burst_active_ms=1616.292896 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:49:16.174753Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:49:16.174958Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:49:16.176388Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:49:16.177121Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:49:16.177128Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:49:16.177131Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121a-real;121] (cubin) +2026-09-21T01:49:16.177132Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:49:16.177134Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:49:16.177135Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:49:16.177136Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:49:16.177138Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:49:16.177139Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:49:16.177140Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:49:16.177141Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:49:16.177143Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:49:16.177144Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:49:22.574397Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.048221000000001 first_bonus_ms=578.43031 first_hidden_ms=0.008352 bind_reset_ms=0.028096 draft_ms=2106.660882 verify_ms=174.18058799999994 target_argmax_sync_ms=2475.014844 logprobs_ms=0.008336 walk_ms=0.055183 hidden_concat_ms=0.122784 rollback_ms=41.68764199999998 decode_ms=4807.330501 +2026-09-21T01:49:22.574711Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5391 +2026-09-21T01:49:22.574750Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5391.33159 burst_active_ms=5391.33159 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:49:25.602329Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.741046 first_bonus_ms=117.01916 first_hidden_ms=0.013776 bind_reset_ms=0.085744 draft_ms=262.36315399999995 verify_ms=184.79203900000005 target_argmax_sync_ms=2407.873518 logprobs_ms=0.005920000000000004 walk_ms=0.04950400000000002 hidden_concat_ms=0.129584 rollback_ms=40.949785999999996 decode_ms=2903.505671 +2026-09-21T01:49:25.602651Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=3023 +2026-09-21T01:49:25.602697Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=3023.87626 burst_active_ms=3023.87626 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.B.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.B.wclassic.log new file mode 100644 index 000000000..505609fbf --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.B.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:52:53.503527Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:52:53.503572Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:52:53.503711Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:52:53.503765Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:52:53.503767Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:52:53.503769Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:52:53.851714Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:52:53.851749Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:52:53.852324Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:52:54.206456Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:52:54.207246Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:52:54.552633Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.345s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.345323009 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:52:54.552709Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:52:54.553133Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:52:54.553147Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:52:54.553171Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:52:54.554288Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:52:55.449075Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 895ms prompt_tokens=1 cached_tokens=0 generation_time_ms=895 +2026-09-21T01:52:55.449183Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:52:55.449272Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:52:55.450243Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:52:55.459759Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:52:55.459777Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:52:55.459780Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121a-real;121] (cubin) +2026-09-21T01:52:55.459782Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:52:55.459783Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:52:55.459785Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:52:55.459786Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:52:55.459787Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:52:55.459789Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:52:55.459790Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:52:55.459791Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:52:55.459792Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:52:55.459794Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:53:00.151698Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3982ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3982 +2026-09-21T01:53:03.650216Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3496ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3496 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.C.w4.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.C.w4.log new file mode 100644 index 000000000..86edf595f --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.C.w4.log @@ -0,0 +1,45 @@ +2026-09-21T01:50:26.915054Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:50:26.915097Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:50:26.915225Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:50:26.915299Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:50:26.915301Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:50:26.915304Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:50:27.255945Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:50:27.255982Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:50:27.256547Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:50:27.601562Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:50:27.602195Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:50:27.947968Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.346s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.345733807 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:50:27.948045Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto, speculative=dflash (drafter=/home/inureyes/models/mlx/qwen3.5-4b-dflash, block_size=4, block_size_from=operator override, explicit_kind=true)) +2026-09-21T01:50:27.948448Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:50:27.948461Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:50:27.948480Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:50:29.556678Z INFO mlxcel::models::speculative_exactness: MTP exactness probe passed: verify block is byte-identical to the single-token chain block_size=4 +2026-09-21T01:50:29.556709Z INFO mlxcel::server::batch::speculative_burst: Lazy-loading drafter from /home/inureyes/models/mlx/qwen3.5-4b-dflash (kind=Some(Dflash)) +2026-09-21T01:50:29.557112Z INFO mlxcel::server::batch::speculative_burst: Drafter loaded (kind=dflash, 0 ms) +2026-09-21T01:50:29.558105Z INFO mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:50:29.580603Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=0 proposed_tokens=0 accepted_tokens=0 acceptance_rate=0.0 emitted_per_verify=0.0 zero_accept_rounds=0 partial_accept_rounds=0 full_accept_rounds=0 prefill_verify_ms=0.947997 first_bonus_ms=22.449173 first_hidden_ms=0.010768 bind_reset_ms=0.047535 draft_ms=0.0 verify_ms=0.0 target_argmax_sync_ms=0.0 logprobs_ms=0.0 walk_ms=0.0 hidden_concat_ms=0.0 rollback_ms=0.0 decode_ms=0.0 +2026-09-21T01:50:29.580788Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=1 generated_tokens=1 burst_ms=1631 +2026-09-21T01:50:29.580794Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-0 burst_wall_ms=1632.092377 burst_active_ms=1632.092377 slices=1 tokens_generated=1 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:50:29.580971Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:50:29.581134Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:50:29.582705Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:50:29.583462Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:50:29.583469Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:50:29.583472Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:50:29.583474Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:50:29.583475Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:50:29.583476Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:50:29.583478Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:50:29.583479Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:50:29.583480Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:50:29.583481Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:50:29.583482Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:50:29.583484Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:50:29.583485Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:50:35.936725Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=5.033763 first_bonus_ms=584.819013 first_hidden_ms=0.008912 bind_reset_ms=0.028352 draft_ms=2069.6160390000005 verify_ms=187.899667 target_argmax_sync_ms=2462.2250029999996 logprobs_ms=0.008496000000000002 walk_ms=0.06158399999999999 hidden_concat_ms=0.12512 rollback_ms=42.87869200000001 decode_ms=4771.318085 +2026-09-21T01:50:35.937425Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=5362 +2026-09-21T01:50:35.937481Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-1 burst_wall_ms=5362.093803 burst_active_ms=5362.093803 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 +2026-09-21T01:50:38.984510Z INFO mlxcel::server::batch::dflash_target: DFlash diagnostics block_size=4 rounds=79 proposed_tokens=237 accepted_tokens=121 acceptance_rate=0.510548523206751 emitted_per_verify=2.518987341772152 zero_accept_rounds=22 partial_accept_rounds=34 full_accept_rounds=23 prefill_verify_ms=2.8376080000000004 first_bonus_ms=120.04775500000001 first_hidden_ms=0.009424 bind_reset_ms=0.068816 draft_ms=264.33038100000005 verify_ms=185.29048299999994 target_argmax_sync_ms=2418.631441 logprobs_ms=0.007647000000000003 walk_ms=0.051744000000000005 hidden_concat_ms=0.12809600000000002 rollback_ms=44.029168 decode_ms=2919.552205 +2026-09-21T01:50:38.984853Z INFO mlxcel::server::batch::speculative_burst: Speculative burst completed prompt_tokens=158 generated_tokens=200 burst_ms=3043 +2026-09-21T01:50:38.984898Z INFO mlxcel::server::batch::scheduler::speculative_finalize: speculative B=1 burst finalized (burst_wall_ms is the max single-tick HOL stall on concurrent rows) seq_id=seq-2 burst_wall_ms=3043.013663 burst_active_ms=3043.013663 slices=1 tokens_generated=200 rounds=0 accepted_draft_tokens=0 hol_waiters=0 diff --git a/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.C.wclassic.log b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.C.wclassic.log new file mode 100644 index 000000000..6727e2400 --- /dev/null +++ b/docs/benchmark_results/data/blackwell-arch-specific-gb10-2026-09-21/logs/server.r5.C.wclassic.log @@ -0,0 +1,36 @@ +2026-09-21T01:54:04.919335Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-09-21T01:54:04.919377Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-09-21T01:54:04.919506Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=1 kv_unified=false n_parallel=4 prefill_chunk_size=512 max_kv_size=None +2026-09-21T01:54:04.919568Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-09-21T01:54:04.919571Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-09-21T01:54:04.919573Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-09-21T01:54:05.246619Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-09-21T01:54:05.246656Z INFO mlxcel::server::startup: Tokenizer recognizes a think marker pair; defaulting chat_template kwarg `enable_thinking=true` (upstream PR #1114) think_start=Some("") think_end=Some("") think_start_tokens_len=1 think_end_tokens_len=1 +2026-09-21T01:54:05.247203Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=536870912 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-09-21T01:54:05.571337Z INFO mlxcel::server::startup: Warming up model... +2026-09-21T01:54:05.571351Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-09-21T01:54:05.927910Z INFO mlxcel::server::model_provider::model_worker: Model qwen3.5-4b-4bit loaded in 0.357s (resident after load: 0.00 GB) worker_model_id=qwen3.5-4b-4bit load_seconds=0.356538341 active_bytes=0 peak_bytes=0 cache_bytes=0 limit_bytes=124128085401 +2026-09-21T01:54:05.927989Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=1, max_queue_depth=32, prefill_chunk_size=512, max_batch_prefill=4, decode_storage=auto) +2026-09-21T01:54:05.928422Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 3062487 blocks (32 layers, 32-token blocks) +2026-09-21T01:54:05.928436Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 256 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-09-21T01:54:05.928457Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=32 kv_cache_mode_total_layers=32 +2026-09-21T01:54:05.929637Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-09-21T01:54:06.872865Z INFO prefill{seq_id=seq-0 prompt_len=1 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/1 prompt tokens, total 944ms prompt_tokens=1 cached_tokens=0 generation_time_ms=944 +2026-09-21T01:54:06.873045Z INFO mlxcel::server::startup: Warmup complete +2026-09-21T01:54:06.873206Z INFO mlxcel::server::startup: model_type=Qwen35VLM: enabling native video_url content block support +2026-09-21T01:54:06.873897Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-09-21T01:54:06.874606Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18934 +2026-09-21T01:54:06.874622Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-09-21T01:54:06.874625Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-09-21T01:54:06.874626Z INFO mlxcel::server::startup: Endpoints: +2026-09-21T01:54:06.874627Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-09-21T01:54:06.874629Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-09-21T01:54:06.874630Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-09-21T01:54:06.874631Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-09-21T01:54:06.874632Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-09-21T01:54:06.874634Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-09-21T01:54:06.874635Z INFO mlxcel::server::startup: GET /props - Server properties +2026-09-21T01:54:06.874636Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-09-21T01:54:06.874637Z INFO mlxcel::server::startup: GET /health - Health check +2026-09-21T01:54:11.578651Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 4002ms prompt_tokens=158 cached_tokens=0 generation_time_ms=4002 +2026-09-21T01:54:15.072427Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/158 prompt tokens, total 3490ms prompt_tokens=158 cached_tokens=0 generation_time_ms=3490 diff --git a/docs/installation.md b/docs/installation.md index da0f9c709..55a8297b9 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -171,18 +171,68 @@ Hopper quantized kernel (`qmm_sm90`) is only compiled when `90a` is in the arch list. An explicitly set `MLX_CUDA_ARCHITECTURES` is used verbatim, so include the suffix yourself for Hopper (`90a`). +On Blackwell (`sm_100`, `sm_120`, `sm_121`) the `a` suffix decides something +else, and you should not set it. MLX compiles the hardware block-float +converters in `mlx/backend/cuda/quantized/nvfp4_quantize.cuh`, the ones that +issue `cvt.rn.satfinite.e2m1x2.f32`, only when nvcc is compiling for an +architecture-specific target, which it signals with `__CUDA_ARCH_SPECIFIC__`. +A plain `121` build therefore compiles them out, and both NVFP4 and MXFP4 +quantization fall back to a scalar CUTLASS conversion sequence with nothing in +the build output saying so. + +Putting `121a` in the list would fix that by charging every translation unit +for it. CUTLASS derives `CUTLASS_ARCH_MMA_SM121A_ENABLED` from the same macro, +so the decode kernels compile differently too, and on a device the suffix +matches, the architecture-specific image is the one the driver loads rather +than an alternative it can ignore. On this project's GB10 that is 12.7 MB of +extra archive and a different `qmv` on every Blackwell machine, to fix a +converter no decode kernel calls. + +So the list stays plain and `src/lib/mlx-cpp/CMakeLists.txt` adds the +architecture-specific image to `fp_quantize.cu` alone, which is the only +translation unit that can reach those converters. You get the hardware path +without asking for it, and without it reaching anything else. Building with +`121a` is a step backwards, not a step forwards; `121f` is worse still, because +it satisfies the converters' dispatcher gate but not their own gate and fails +to compile outright. + ```bash # Hopper / GH200-style target. The `a` suffix is required for the Hopper # quantized kernel; plain `90` builds without it. MLX_CUDA_ARCHITECTURES=90a cargo build --release --features cuda -# GB10 / DGX Spark-style target used by the release workflow. +# GB10 / DGX Spark-style target used by the release workflow. Plain: the +# hardware NVFP4/MXFP4 converter is added to one translation unit by CMake. MLX_CUDA_ARCHITECTURES=121 cargo build --release --features cuda # Multiple targets, if your MLX/CUDA toolchain supports them. MLX_CUDA_ARCHITECTURES="90a;121" cargo build --release --features cuda ``` +To confirm what a finished build carries, read the archive rather than the +environment variable. A correct Blackwell build has the fp4 instruction in +`fp_quantize.cu.o`, exactly one architecture-specific image in the whole +archive, and its forward-JIT PTX untouched: + +```bash +A=$(ls -d target/release/build/mlxcel-core-*/out/build/lib/libmlx.a | head -1) +mkdir -p /tmp/mlxq && (cd /tmp/mlxq && ar x "$A" fp_quantize.cu.o qmv.cu.o) +cuobjdump --dump-sass /tmp/mlxq/fp_quantize.cu.o | grep -c F2FP.SATFINITE.E2M1 # > 0 +cuobjdump --list-elf /tmp/mlxq/qmv.cu.o | grep -c 'sm_[0-9]*a' # 0: decode untouched +cuobjdump --list-elf "$A" | grep -c 'sm_[0-9]*a' # 1 +cuobjdump --dump-ptx "$A" | grep -c '.target sm_121$' # > 0 +``` + +One checkout can hold several of those build directories, one per feature set +and architecture list, so confirm the one you are reading is the one you meant: +`grep MLX_CUDA_ARCHITECTURES "$(dirname "$A")/../CMakeCache.txt"` names the list +it was configured with. + +Do not look for `cvt.rn.satfinite.e2m1x2` in the PTX. The injection emits a +cubin and no PTX, and the PTX that is emitted comes from the plain `compute_121` +pass, which takes the fallback arm, so that grep returns zero on a correct +build. The SASS count above is the one that distinguishes the two builds. + If the architecture list a binary was built with does not cover the GPU it is started on, it refuses to start and says so, naming both the list it carries and the compute capability it found, instead of failing later with an opaque CUDA @@ -201,7 +251,9 @@ wrong architecture can still be run on the CPU while a correct one is built. The repository release workflow builds two Linux CUDA targets on self-hosted runners, each as one fat binary: aarch64 covering GH200 (`90a`), GB200 (`100`), and GB10 (`121`) in a single build (`90a;100;121`), and x86_64 covering Ampere -through Blackwell (`80;86;89;90a;100;120`). For each target the `mlxcel` CLI and the +through Blackwell (`80;86;89;90a;100;120`). Blackwell is plain in both for the +reason above, with the hardware converter added to one translation unit by +CMake. For each target the `mlxcel` CLI and the `mlxcel-server` are published as separate archives (`mlxcel-...` and `mlxcel-server-...`, each roughly 347 MB) so a consumer downloads only the one it needs. Every published release also ships a CycloneDX SBOM named diff --git a/scripts/ci/check_cuda_arch_lists.py b/scripts/ci/check_cuda_arch_lists.py new file mode 100755 index 000000000..84228a934 --- /dev/null +++ b/scripts/ci/check_cuda_arch_lists.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Keep Blackwell architecture lists plain, and keep the mechanism that makes +that safe in place. + +Rationale +--------- +MLX compiles its hardware block-float converters (``cvt.rn.satfinite.e2m1x2.f32`` +and siblings in ``mlx/backend/cuda/quantized/nvfp4_quantize.cuh``) only when the +translation unit is compiled for an *architecture-specific* target, which nvcc +signals with ``__CUDA_ARCH_SPECIFIC__`` and CMake spells as the ``a`` suffix +(``121a``). Every list this repository ships names Blackwell plainly, so on its +own that would compile the converters out of every artifact and every test, +which is the defect issue #1934 opened on. + +The obvious fix, naming Blackwell ``121a`` in these lists, buys the converter by +charging every translation unit for it: CUTLASS derives +``CUTLASS_ARCH_MMA_SM121A_ENABLED`` from the same macro, so the decode kernels +compile differently too, and on a device the suffix matches, the +architecture-specific image is the one the driver loads rather than an +alternative it may ignore. Measured on GB10, that is 12.7 MB of extra archive +and a different ``qmv`` on every Blackwell user's machine, to fix a converter +none of those kernels call. + +``src/lib/mlx-cpp/CMakeLists.txt`` adds the architecture-specific image to +``fp_quantize.cu`` alone instead, which is the only translation unit that can +reach the converters. So these lists stay plain, and an ``a`` in one is now a +regression rather than a fix. + +The rules +--------- +For every ``MLX_CUDA_ARCHITECTURES`` value in ``.github/workflows/*.yml``: + +1. Every entry must parse as CMake spells one: ``[a|f][-real|-virtual]``. +2. No entry may use the family-specific ``f`` suffix. ``sm_121f`` satisfies the + dispatcher gate in ``nvfp4_quantize.cuh`` (``__CUDA_ARCH_FAMILY_SPECIFIC__ >= + 1000``) but not the converter gate (``__CUDA_ARCH_SPECIFIC__``), so the fast + path calls converters that were never defined and nvcc fails with three + errors in that header. It is rejected here rather than discovered in a + 40-minute CUDA job. +3. No Blackwell entry (major >= 10) may carry the ``a`` suffix, because that is + the whole-target form the per-source injection exists to avoid. Hopper's + ``90a`` is untouched: it is load-bearing for MLX's own quantized-kernel gate + and cannot reach these converters anyway, which also require compute + capability 10.0. + +Then, once for the repository: + +4. ``src/lib/mlx-cpp/CMakeLists.txt`` must still carry the per-source injection. + Without it, plain lists mean the converters are compiled out again, silently, + which is exactly the state this issue started from. +5. Every list in ``release.yml`` must appear verbatim in + ``docs/installation.md``. Documentation drift is how the original defect + stayed invisible: the document names the architectures the published archives + carry, and nothing made it move when the workflow did. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +#: The first Blackwell compute capability major version. The converter gate in +#: ``nvfp4_quantize.cuh`` is ``__CUDA_ARCH__ >= 1000``, i.e. compute capability +#: 10.0 and newer, which is exactly "Blackwell and later". +BLACKWELL_MAJOR = 10 + +#: ``env:`` assignment of the architecture list, in any workflow, at any indent. +ENV_ASSIGNMENT = re.compile(r"^\s*MLX_CUDA_ARCHITECTURES:\s*(?P\S.*?)\s*$") + +#: One CMake architecture-list entry: digits, an optional variant suffix, an +#: optional code-object restriction. Mirrors ``parse_cuda_arch_entry`` in +#: ``src/lib/mlxcel-core/src/cuda_arch.rs``, which parses the same strings at +#: runtime for the startup mismatch check. +ENTRY = re.compile(r"^(?P\d{2,})(?P[af]?)(?P-real|-virtual)?$") + + +@dataclass(frozen=True) +class Entry: + """One parsed architecture-list entry.""" + + raw: str + major: int + minor: int + variant: str # "", "a" or "f" + emits_cubin: bool + emits_ptx: bool + + @property + def capability(self) -> tuple[int, int]: + return (self.major, self.minor) + + +@dataclass(frozen=True) +class ArchList: + """One ``MLX_CUDA_ARCHITECTURES`` assignment, located.""" + + path: Path + line: int + value: str + + +def _parse_entry(raw: str) -> Entry | None: + match = ENTRY.match(raw.strip()) + if match is None: + return None + digits = match.group("digits") + restriction = match.group("restriction") + # The minor version is the last digit: 70 is 7.0 and 121 is 12.1. + return Entry( + raw=raw.strip(), + major=int(digits[:-1]), + minor=int(digits[-1]), + variant=match.group("variant"), + emits_cubin=restriction != "-virtual", + emits_ptx=restriction != "-real", + ) + + +def _strip_yaml_scalar(value: str) -> str: + """Return the string a YAML plain or quoted scalar denotes. + + Only the two shapes these workflows use are handled, because anything else + in this position should be read by a human before it reaches nvcc. + """ + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + # A plain scalar can carry a trailing comment; a quoted one cannot. + return value.split("#", 1)[0].strip() + + +def collect_arch_lists(workflow_dir: Path) -> list[ArchList]: + """Every architecture-list assignment in every workflow, in file order.""" + found: list[ArchList] = [] + for path in sorted(workflow_dir.glob("*.yml")) + sorted(workflow_dir.glob("*.yaml")): + for number, line in enumerate(path.read_text().splitlines(), start=1): + match = ENV_ASSIGNMENT.match(line) + if match is None: + continue + found.append(ArchList(path=path, line=number, value=_strip_yaml_scalar(match.group("value")))) + return found + + +def check_list(arch_list: ArchList) -> list[str]: + """Every rule violation in one list, as operator-readable lines.""" + problems: list[str] = [] + entries: list[Entry] = [] + for raw in arch_list.value.split(";"): + if not raw.strip(): + continue + entry = _parse_entry(raw) + if entry is None: + problems.append( + f"entry {raw.strip()!r} is not a CMake architecture spelling " + "([a|f][-real|-virtual], e.g. 121a-real)" + ) + continue + entries.append(entry) + + for entry in entries: + if entry.variant == "f": + problems.append( + f"entry {entry.raw!r} uses the family-specific `f` suffix, which does not compile: " + "it satisfies the dispatcher gate in mlx/backend/cuda/quantized/nvfp4_quantize.cuh " + "(__CUDA_ARCH_FAMILY_SPECIFIC__ >= 1000) but not the converter gate " + "(__CUDA_ARCH_SPECIFIC__), so the fast path calls converters that were never " + "defined and nvcc fails with three errors in that header. Use the `a` suffix." + ) + + for entry in entries: + if entry.variant != "a" or entry.major < BLACKWELL_MAJOR: + continue + sm = f"{entry.major}{entry.minor}" + problems.append( + f"Blackwell entry {entry.raw!r} is architecture-specific at the target level. That " + "compiles every translation unit with __CUDA_ARCH_SPECIFIC__, not just the one that " + "reaches MLX's hardware NVFP4/MXFP4 converters: CUTLASS keys " + "CUTLASS_ARCH_MMA_SM121A_ENABLED on the same macro, so the decode kernels get a " + "second image, and on a matching device that is the image the driver loads. Measured " + "on GB10 it costs 12.7 MB of archive to change code no converter calls. Use plain " + f"{sm}; src/lib/mlx-cpp/CMakeLists.txt gives fp_quantize.cu the {sm}a image on its " + "own. See issue #1934." + ) + + return problems + + +def check_injection(cmakelists: Path) -> list[str]: + """The per-source injection that lets the lists above stay plain.""" + if not cmakelists.exists(): + return [f"{cmakelists} does not exist, so the per-source injection cannot be checked"] + text = cmakelists.read_text() + required = ("set_source_files_properties", "fp_quantize.cu", "TARGET_DIRECTORY mlx") + missing = [token for token in required if token not in text] + if not missing: + return [] + return [ + f"{cmakelists} no longer carries the per-source architecture injection (missing " + f"{', '.join(repr(m) for m in missing)}). Without it the plain Blackwell lists above " + "compile MLX's hardware NVFP4/MXFP4 converters out of every artifact and every test, " + "silently, which is the defect issue #1934 was opened on." + ] + + +def check_docs(release_lists: list[ArchList], installation: Path) -> list[str]: + """Release lists that `docs/installation.md` does not quote verbatim.""" + if not installation.exists(): + return [f"{installation} does not exist, so the release lists cannot be checked against it"] + text = installation.read_text() + return [ + f"{arch_list.path.name}:{arch_list.line} builds {arch_list.value!r}, which " + f"{installation.name} does not name. The document tells operators which architectures the " + "published archives carry; a list that moves without it is how issue #1934 stayed " + "invisible. Quote the list verbatim there." + for arch_list in release_lists + if arch_list.value not in text + ] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "--root", + type=Path, + default=REPO_ROOT, + help="repository root to check (default: this script's repository)", + ) + args = parser.parse_args() + + workflow_dir = args.root / ".github" / "workflows" + if not workflow_dir.is_dir(): + print(f"no workflow directory at {workflow_dir}", file=sys.stderr) + return 1 + + arch_lists = collect_arch_lists(workflow_dir) + if not arch_lists: + print( + f"no MLX_CUDA_ARCHITECTURES assignment found under {workflow_dir}. Either the CUDA " + "jobs lost their architecture pin (auto-detection then decides what ships) or this " + "check no longer knows where to look.", + file=sys.stderr, + ) + return 1 + + failures: list[str] = [] + for arch_list in arch_lists: + for problem in check_list(arch_list): + relative = arch_list.path.relative_to(args.root) + failures.append(f"{relative}:{arch_list.line}: {problem}") + + failures.extend(check_injection(args.root / "src" / "lib" / "mlx-cpp" / "CMakeLists.txt")) + release_lists = [a for a in arch_lists if a.path.name == "release.yml"] + failures.extend(check_docs(release_lists, args.root / "docs" / "installation.md")) + + if failures: + print("CUDA architecture lists are not shippable:", file=sys.stderr) + for failure in failures: + print(f" {failure}", file=sys.stderr) + return 1 + + print(f"CUDA architecture lists OK ({len(arch_lists)} checked):") + for arch_list in arch_lists: + print(f" {arch_list.path.relative_to(args.root)}:{arch_list.line} {arch_list.value}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/check_cuda_arch_lists_test.sh b/scripts/ci/check_cuda_arch_lists_test.sh new file mode 100755 index 000000000..6f4c3d336 --- /dev/null +++ b/scripts/ci/check_cuda_arch_lists_test.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Deterministic coverage for scripts/ci/check_cuda_arch_lists.py. +# +# A guard that only ever passes proves nothing. Each case below builds a +# throwaway root holding one workflow, one installation document and one +# CMakeLists, runs the checker against it with --root, and asserts both the +# exit status and the line the operator is supposed to read. +# +# The case that matters most is the architecture-specific Blackwell list. That +# is the obvious fix for issue #1934 and the one an earlier revision of that +# issue's own PR shipped: it works, and it charges every translation unit for a +# converter only one of them reaches. The `121f` case is the second most likely +# wrong turn, since `f` reads as a milder `a` and does not compile at all. + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +under_test="$script_dir/check_cuda_arch_lists.py" + +work="$(mktemp -d "${TMPDIR:-/tmp}/cuda-arch-lists-test.XXXXXX")" +trap 'rm -rf "$work"' EXIT + +failures=0 + +# Builds a root with one workflow carrying $2 as its architecture list, and an +# installation document quoting $3 (default: the same list, so a case that is +# not about documentation drift never trips the documentation rule). +setup_root() { + local root="$1" list="$2" documented="${3-$2}" injection="${4-present}" + mkdir -p "$root/.github/workflows" "$root/docs" "$root/src/lib/mlx-cpp" + cat > "$root/.github/workflows/release.yml" < "$root/docs/installation.md" + if [ "$injection" = "present" ]; then + cat > "$root/src/lib/mlx-cpp/CMakeLists.txt" <<'CM' +set_source_files_properties( + "${mlx_SOURCE_DIR}/mlx/backend/cuda/quantized/fp_quantize.cu" + TARGET_DIRECTORY mlx PROPERTIES COMPILE_OPTIONS "${_fp_quantize_arch_flags}") +CM + else + printf '# the injection was removed\n' > "$root/src/lib/mlx-cpp/CMakeLists.txt" + fi +} + +# Runs the checker and asserts the exit status and, when given, that the output +# contains an expected fragment. +expect() { + local name="$1" root="$2" want_status="$3" want_text="${4-}" + local output status + set +e + output="$(python3 "$under_test" --root "$root" 2>&1)" + status=$? + set -e + if [ "$status" -ne "$want_status" ]; then + echo "FAIL: $name: expected exit $want_status, got $status" >&2 + echo "$output" | sed 's/^/ /' >&2 + failures=$((failures + 1)) + return + fi + if [ -n "$want_text" ] && ! printf '%s' "$output" | grep -qF -- "$want_text"; then + echo "FAIL: $name: output did not mention '$want_text'" >&2 + echo "$output" | sed 's/^/ /' >&2 + failures=$((failures + 1)) + return + fi + echo "ok: $name" +} + +# 1. What release.yml ships, and what it must keep shipping. +setup_root "$work/plain" "90a;100;121" +expect "plain Blackwell list passes" "$work/plain" 0 "CUDA architecture lists OK" + +# 2. `f` compiles neither here nor in CI; rejecting it by name saves a CUDA job. +setup_root "$work/family" "121f;121" +expect "family-specific f is rejected" "$work/family" 1 "does not compile" + +# 3. The whole-target form an earlier revision of #1938 shipped. It compiles the +# converter and charges every other kernel for it. +setup_root "$work/combined" "90a;100a-real;100;121a-real;121" +expect "architecture-specific Blackwell fails" "$work/combined" 1 "architecture-specific at the target level" + +# 4. The same mistake in its simplest spelling. +setup_root "$work/aonly" "90a;121a" +expect "bare 121a fails" "$work/aonly" 1 "architecture-specific at the target level" + +# 5. Hopper keeps its suffix: it is load-bearing for MLX's own quantized-kernel +# gate and cannot reach these converters anyway. +setup_root "$work/hopper" "90a" +expect "Hopper 90a is untouched" "$work/hopper" 0 "CUDA architecture lists OK" + +# 6. Pre-Blackwell lists are untouched: Hopper fails the converter's +# __CUDA_ARCH__ >= 1000 gate however it is spelled. +setup_root "$work/ampere" "70" +expect "pre-Blackwell list passes untouched" "$work/ampere" 0 "CUDA architecture lists OK" + +# 7. A spelling nvcc would reject must not reach nvcc. +setup_root "$work/garbage" "sm_121a" +expect "unparseable entry fails" "$work/garbage" 1 "not a CMake architecture spelling" + +# 8. Documentation drift, which is how the plain-only lists stayed invisible. +setup_root "$work/docs" "121a-real;121" "121" +expect "release list absent from installation.md fails" "$work/docs" 1 "does not name" + +# 9. The per-source injection is what lets the plain lists above be correct. +# Without it they are the original defect again, silently. +setup_root "$work/noinject" "90a;100;121" "90a;100;121" "absent" +expect "missing per-source injection fails" "$work/noinject" 1 "no longer carries the per-source architecture injection" + +# 10. A workflow set that pins nothing hands the decision to auto-detection, so +# what ships stops being reviewable at all. +setup_root "$work/empty" "placeholder" +trash-put "$work/empty/.github/workflows/release.yml" 2>/dev/null || rm -f "$work/empty/.github/workflows/release.yml" +printf 'jobs:\n build:\n runs-on: ubuntu-latest\n' > "$work/empty/.github/workflows/ci.yml" +expect "no architecture pin at all fails" "$work/empty" 1 "no MLX_CUDA_ARCHITECTURES assignment" + +if [ "$failures" -ne 0 ]; then + echo "$failures case(s) failed" >&2 + exit 1 +fi +echo "all cases passed" diff --git a/src/lib/mlx-cpp/CMakeLists.txt b/src/lib/mlx-cpp/CMakeLists.txt index ff1611815..133b53368 100644 --- a/src/lib/mlx-cpp/CMakeLists.txt +++ b/src/lib/mlx-cpp/CMakeLists.txt @@ -233,4 +233,83 @@ if(MLX_BUILD_CUDA) target_compile_definitions(mlx PRIVATE MLX_CUTLASS_DIR="${_cutlass_include}") endif() + + # ------------------- Hardware block-float converter ---------------------- + # MLX compiles its hardware NVFP4/MXFP4 converters, the + # `cvt.rn.satfinite.e2m1x2.f32` family in + # mlx/backend/cuda/quantized/nvfp4_quantize.cuh, only when nvcc is compiling + # for an architecture-specific target, which it signals with + # `__CUDA_ARCH_SPECIFIC__` and which CMake spells as the `a` suffix. Every + # architecture list this project shipped named Blackwell plainly, so those + # converters were compiled out of every release artifact and every test, and + # block-float quantization silently took a scalar CUTLASS fallback (#1934). + # + # Naming the whole target `a` fixes that and charges every kernel for it. + # CUTLASS derives `CUTLASS_ARCH_MMA_SM121A_ENABLED` from the same macro, so + # the decode kernels compile differently too, and on a device the suffix + # matches, the architecture-specific image is the one the driver loads rather + # than an alternative it can ignore. + # + # `fp_quantize.cu` is the only translation unit that can reach the + # converters: it is the only file that includes `fp_quantize.cuh`, which is + # the only file that includes `nvfp4_quantize.cuh`. The gated functions are + # `__device__ __forceinline__` in a header, so they stay inside whatever TU + # instantiates them; the TU exports only ordinary host entry points + # (`mlx::core::fp_quantize` and siblings); and MLX compiles whole-program, + # with no `-rdc`, so no device symbol has to resolve across objects. The + # extra architecture therefore belongs on that one source rather than on the + # target, which is what this does. Every other object, decode included, keeps + # byte for byte the code it had. + set(_fp_quantize + "${mlx_SOURCE_DIR}/mlx/backend/cuda/quantized/fp_quantize.cu") + # + # A capability the list already names architecture-specific needs nothing + # added: the whole target is compiled that way for it, and asking nvcc for the + # same `--generate-code` twice is an error rather than a no-op. This is not + # hypothetical, it is what an operator following the pre-#1934 advice to build + # `121a-real;121` by hand would hit. + set(_already_arch_specific "") + foreach(_entry IN LISTS MLX_CUDA_ARCHITECTURES) + if(_entry MATCHES "^([0-9][0-9]+)a") + list(APPEND _already_arch_specific "${CMAKE_MATCH_1}") + endif() + endforeach() + set(_fp_quantize_arch_flags "") + foreach(_entry IN LISTS MLX_CUDA_ARCHITECTURES) + # Plain Blackwell entries only. An entry that already carries `a` or `f`, + # or a `-real`/`-virtual` restriction, is left exactly as the operator + # wrote it, and Hopper is skipped because these converters also gate on + # `__CUDA_ARCH__ >= 1000`, which 9.0 fails however it is spelled. + # + # `list(FIND)` rather than `IN_LIST`, which needs policy CMP0057 and so + # depends on the `cmake_minimum_required` of whoever includes this file. + if(_entry MATCHES "^([0-9][0-9]+)$") + set(_sm "${CMAKE_MATCH_1}") + list(FIND _already_arch_specific "${_sm}" _sm_index) + if(_sm GREATER_EQUAL 100 AND _sm_index EQUAL -1) + list(APPEND _fp_quantize_arch_flags + "--generate-code=arch=compute_${_sm}a,code=[sm_${_sm}a]") + endif() + endif() + endforeach() + if(_fp_quantize_arch_flags) + if(NOT EXISTS "${_fp_quantize}") + # Silence here is the original defect: the capability disappears and + # nothing says so. A pin bump that moves or renames this file has to + # update the path rather than quietly lose the hardware converter. + message( + FATAL_ERROR + "MLX_CUDA_ARCHITECTURES names a Blackwell target but ${_fp_quantize} " + "does not exist, so MLX's hardware NVFP4/MXFP4 converter would be " + "compiled out (lablup/mlxcel#1934). Update this path for the current " + "MLX pin.") + endif() + set_source_files_properties( + "${_fp_quantize}" TARGET_DIRECTORY mlx PROPERTIES COMPILE_OPTIONS + "${_fp_quantize_arch_flags}") + message( + STATUS + "Hardware block-float converter: fp_quantize.cu also compiled for " + "${_fp_quantize_arch_flags}") + endif() endif() diff --git a/src/lib/mlxcel-core/src/cuda_arch.rs b/src/lib/mlxcel-core/src/cuda_arch.rs index 5ac204115..31d0b4cdd 100644 --- a/src/lib/mlxcel-core/src/cuda_arch.rs +++ b/src/lib/mlxcel-core/src/cuda_arch.rs @@ -47,7 +47,7 @@ use std::sync::OnceLock; /// The `MLX_CUDA_ARCHITECTURES` list this binary's MLX device code was /// compiled for, as a CMake-style semicolon-separated string -/// (`"80;86;89;90a;100;120"`). +/// (`"80;86;89;90a;100a-real;100;120a-real;120"`). /// /// Empty on any build without the `cuda` feature, and empty on a CUDA build /// whose build script predates this record. Callers treat empty as "unknown", diff --git a/src/lib/mlxcel-core/src/cuda_arch_tests.rs b/src/lib/mlxcel-core/src/cuda_arch_tests.rs index 189a0ad2d..c4272c952 100644 --- a/src/lib/mlxcel-core/src/cuda_arch_tests.rs +++ b/src/lib/mlxcel-core/src/cuda_arch_tests.rs @@ -27,9 +27,20 @@ use crate::cuda_arch::{ }; /// The two lists `.github/workflows/release.yml` builds, and the CI pin. +/// +/// Blackwell is plain in both. MLX's hardware NVFP4/MXFP4 converters do need an +/// architecture-specific target, but `src/lib/mlx-cpp/CMakeLists.txt` gives that +/// to `fp_quantize.cu` alone rather than to the whole `mlx` target, so these +/// lists stay plain and every other kernel keeps the code it had (issue #1934). +/// Hopper's `90a` is unrelated and load-bearing for MLX's own quantized-kernel +/// gate. const RELEASE_AARCH64: &str = "90a;100;121"; const RELEASE_X86_64: &str = "80;86;89;90a;100;120"; +/// The Blackwell list every CUDA job in `.github/workflows/ci.yml` pins, and +/// what a GB10 release build carries. +const CI_BLACKWELL: &str = "121"; + /// Compute capabilities of the machines this project actually runs on. const V100: (u32, u32) = (7, 0); const A100: (u32, u32) = (8, 0); @@ -113,8 +124,63 @@ fn parses_the_release_matrices_completely() { assert_eq!(x86_64.len(), 6); // Round-tripping through Display keeps the list readable in the error // message and proves nothing was silently reinterpreted. - let rendered: Vec = x86_64.iter().map(ToString::to_string).collect(); - assert_eq!(rendered.join(";"), RELEASE_X86_64); + for list in [RELEASE_X86_64, RELEASE_AARCH64, CI_BLACKWELL] { + let rendered: Vec = parse_cuda_arch_list(list) + .iter() + .map(ToString::to_string) + .collect(); + assert_eq!(rendered.join(";"), list); + } +} + +/// What the release lists would have meant had Blackwell been named +/// architecture-specific, which is the form issue #1934 measured and rejected. +/// +/// Kept as a test rather than a comment because the coverage semantics are the +/// reason the rejected form is not simply worse-but-equivalent: `121a` alone +/// strands a later Blackwell revision, and `121a-real;121` does not. Neither is +/// needed now that `fp_quantize.cu` carries the architecture-specific image on +/// its own, but a future reader reaching for one of them should find the +/// difference between them written down. +#[test] +fn the_rejected_architecture_specific_blackwell_forms_still_parse_as_documented() { + // The `a-real` half is what would have compiled MLX's hardware fp4 + // converter, and it covers exactly its own target: a cubin, no PTX, + // nothing forward. + let arch_specific = parse_cuda_arch_entry("121a-real").expect("121a-real is a valid entry"); + assert_eq!(arch_specific.variant, ArchVariant::ArchSpecific); + assert!(arch_specific.emits_cubin); + assert!(!arch_specific.emits_ptx); + assert_eq!( + entry_coverage(arch_specific, GB10), + Some(ArchCoverage::Cubin) + ); + assert_eq!(entry_coverage(arch_specific, (13, 0)), None); + + // A plain entry is what survives onto a device the cubin cannot serve, and + // it is what the release lists carry. The difference between the two + // rejected forms is exactly this: `121a-real;121` keeps it, `121a` does not. + assert_eq!( + arch_list_coverage(CI_BLACKWELL, GB10), + Some(ArchCoverage::Cubin) + ); + assert_eq!( + arch_list_coverage(CI_BLACKWELL, (13, 0)), + Some(ArchCoverage::Ptx) + ); + assert_eq!( + arch_list_coverage("121a-real;121", (13, 0)), + Some(ArchCoverage::Ptx) + ); + assert_eq!(arch_list_coverage("121a", (13, 0)), None); + + // Same shape on the x86_64 Blackwell entries: `100a-real` is pinned to 10.0 + // while plain `100` still carries the major version forward by cubin. + assert_eq!( + arch_list_coverage("100a-real;100", (10, 3)), + Some(ArchCoverage::Cubin) + ); + assert_eq!(arch_list_coverage("100a-real", (10, 3)), None); } #[test] @@ -258,7 +324,16 @@ fn an_empty_or_unparseable_list_covers_nothing() { #[test] fn entry_display_round_trips_through_the_parser() { - for spelling in ["70", "90a", "100f", "80-real", "89-virtual", "121"] { + for spelling in [ + "70", + "90a", + "100f", + "80-real", + "89-virtual", + "121", + "121a-real", + "120a-real", + ] { let entry = parse_cuda_arch_entry(spelling).expect("valid entry"); assert_eq!(entry.to_string(), spelling); assert_eq!(parse_cuda_arch_entry(&entry.to_string()), Some(entry));