diff --git a/docs/maxusai/tasks/mmq-29953-guard-verify.sh b/docs/maxusai/tasks/mmq-29953-guard-verify.sh index ca1c10f4bea..bd5b22f495e 100755 --- a/docs/maxusai/tasks/mmq-29953-guard-verify.sh +++ b/docs/maxusai/tasks/mmq-29953-guard-verify.sh @@ -12,6 +12,13 @@ # 5. one blackwell config at 192 threads-> must FAIL by the guard naming ONLY blackwell: GGML_PAD is a bit mask # that assumes a power-of-two granularity, the tables only assert # nthreads % 32 == 0, and the guard's exact arithmetic sees the gap +# 5b. one rdna3_5 config at 160 threads -> must FAIL by the guard naming ONLY rdna3_5. Not 192: rdna3_5's first +# 256-thread line has J = 64, and 64*36 ints is a multiple of 192, so the +# bit mask happens to round right there (found by the ROCm host, #455) +# +# The guard's outer check is an explicitly instantiated class inheriting every per-type guard, not a static_assert +# on a fold: under clang a failed per-type assertion made that outer expression non-constant and added one error +# per failing table, worded like a constant-evaluation limit. The ROCm host's variant (#455, #456). # nvcc's front end stops at 100 errors by default, which cut mutation 3's table list to the first five tables; # --error_limit lifts that so every table that fails is named. # Usage: mmq-29953-guard-verify.sh (from its own directory, with a llama.cpp clone at ./llama.cpp that has @@ -67,6 +74,7 @@ build "1-head-unpatched" ok || ok=0 git apply "$PATCH" || { echo "FAIL: the patch does not apply to $HEAD29953"; exit 1; } echo "patch applies to a pristine $HEAD29953" cp "$CU" "$OUT/mmq.cu.patched"; cp "$CUH" "$OUT/mmq.cuh.patched"; cp "$BW" "$OUT/blackwell.orig" +R35=ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh; cp "$R35" "$OUT/rdna3-5.orig" build "2-head-with-guard" ok || ok=0 swap() { # : exact one-occurrence string swap, so a mutation cannot fail for a typo of mine @@ -96,6 +104,12 @@ echo "mutation 5 rewrote: $first -> ${first/256,/192,}" build "5-blackwell-config-at-192-threads" fail "blackwell" || ok=0 cp "$OUT/blackwell.orig" "$BW" +first=$(grep -m1 -E '^ *CASE\(GGML_TYPE_[A-Z0-9_]+, *256,' "$R35") +swap "$R35" "$first" "${first/256,/160,}" +echo "mutation 5b rewrote: $first -> ${first/256,/160,}" +build "5b-rdna3_5-config-at-160-threads" fail "rdna3_5" || ok=0 +cp "$OUT/rdna3-5.orig" "$R35" + sed -i 's/^# define GGML_CUDA_MMQ_Y_TILE_GUARD_ON$/\/\/ guard disabled for timing/; s/^# define GGML_CUDA_MMQ_Y_TILE_GUARD_ON$/\/\/ guard disabled for timing/' "$CU" build "6-guard-compiled-out" ok || ok=0 diff --git a/docs/maxusai/tasks/mmq-29953-y-tile-guard.patch b/docs/maxusai/tasks/mmq-29953-y-tile-guard.patch index 729f22c2727..3f4c306ab0a 100644 --- a/docs/maxusai/tasks/mmq-29953-y-tile-guard.patch +++ b/docs/maxusai/tasks/mmq-29953-y-tile-guard.patch @@ -1,5 +1,5 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu -index 027a7e60..6923d84a 100644 +index 027a7e60..fcfaa3c4 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -4,6 +4,7 @@ @@ -84,10 +84,10 @@ index 027a7e60..6923d84a 100644 + static_assert(ggml_cuda_mmq_y_tile_ok_##TABLE(t), \ + "MMQ: the y tile helper is smaller than the y load for a config in the " #TABLE " table"); \ + }; \ -+ template static constexpr size_t ggml_cuda_mmq_y_tile_all_##TABLE(std::integer_sequence) { \ -+ return (sizeof(ggml_cuda_mmq_y_tile_guard_##TABLE) + ... + 0); \ -+ } \ -+ static_assert(ggml_cuda_mmq_y_tile_all_##TABLE(std::make_integer_sequence()) > 0, ""); ++ template struct ggml_cuda_mmq_y_tile_guards_##TABLE; \ ++ template struct ggml_cuda_mmq_y_tile_guards_##TABLE> \ ++ : ggml_cuda_mmq_y_tile_guard_##TABLE... {}; \ ++ template struct ggml_cuda_mmq_y_tile_guards_##TABLE>; + +GGML_CUDA_MMQ_Y_TILE_GUARD(pascal_older) +GGML_CUDA_MMQ_Y_TILE_GUARD(pascal_dp4a) diff --git a/docs/maxusai/tasks/mmq-successor-results/guard-timing.txt b/docs/maxusai/tasks/mmq-successor-results/guard-timing.txt index b821199acd9..ce8c70d86cc 100644 --- a/docs/maxusai/tasks/mmq-successor-results/guard-timing.txt +++ b/docs/maxusai/tasks/mmq-successor-results/guard-timing.txt @@ -1,8 +1,8 @@ -host load at start (1/5/15 min): 6.96 11.67 17.05, 32 cores -rep 1: guard on 11.09s, off 8.15s -rep 2: guard on 10.91s, off 8.27s -rep 3: guard on 10.90s, off 7.86s -rep 4: guard on 11.26s, off 8.59s -rep 5: guard on 11.90s, off 7.95s -median: guard on 11.09s, off 8.15s, over 5 interleaved reps -host load at end (1/5/15 min): 8.10 10.59 16.12 +host load at start (1/5/15 min): 28.86 36.27 31.15, 32 cores +rep 1: guard on 12.44s, off 10.74s +rep 2: guard on 15.17s, off 10.46s +rep 3: guard on 15.10s, off 12.04s +rep 4: guard on 16.77s, off 12.05s +rep 5: guard on 16.99s, off 11.75s +median: guard on 15.17s, off 11.75s, over 5 interleaved reps +host load at end (1/5/15 min): 27.44 32.46 30.37 diff --git a/docs/maxusai/tasks/mmq-successor-results/verify-29953-guard.txt b/docs/maxusai/tasks/mmq-successor-results/verify-29953-guard.txt index 29b2632f130..275aeeb7b4f 100644 --- a/docs/maxusai/tasks/mmq-successor-results/verify-29953-guard.txt +++ b/docs/maxusai/tasks/mmq-successor-results/verify-29953-guard.txt @@ -1,13 +1,16 @@ -host load at start (1/5/15 min): 7.73 12.80 17.77, 32 cores -1-head-unpatched rc=0 ok want=ok as-expected 8.1s +host load at start (1/5/15 min): 38.78 35.94 28.60, 32 cores +1-head-unpatched rc=0 ok want=ok as-expected 13.5s patch applies to a pristine 3070d927f6c172242d56a2c1fb897e838e7b2632 -2-head-with-guard rc=0 ok want=ok as-expected 11.2s -3-helper-shrunk-to-J-blocks rc=2 fail want=fail as-expected 3.0s +2-head-with-guard rc=0 ok want=ok as-expected 19.1s +3-helper-shrunk-to-J-blocks rc=2 fail want=fail as-expected 5.0s errors=220, from the guard=220, tables named: ampere blackwell cdna gcn pascal_dp4a pascal_older rdna2 rdna3 rdna3_5 rdna4 -4-helper-rounded-to-one-warp rc=2 fail want=fail as-expected 3.0s +4-helper-rounded-to-one-warp rc=2 fail want=fail as-expected 5.0s errors=220, from the guard=220, tables named: ampere blackwell cdna gcn pascal_dp4a pascal_older rdna2 rdna3 rdna3_5 rdna4 mutation 5 rewrote: CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); -> CASE(GGML_TYPE_MXFP4, 192, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); -5-blackwell-config-at-192-threads rc=2 fail want=fail as-expected 4.0s +5-blackwell-config-at-192-threads rc=2 fail want=fail as-expected 6.1s errors=1, from the guard=1, tables named: blackwell -6-guard-compiled-out rc=0 ok want=ok as-expected 7.9s -VERIFY29953GUARD COMPLETE 2026-10-05T02:33:16Z all_as_expected=1 +mutation 5b rewrote: CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); -> CASE(GGML_TYPE_Q1_0, 160, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); +5b-rdna3_5-config-at-160-threads rc=2 fail want=fail as-expected 6.0s + errors=1, from the guard=1, tables named: rdna3_5 +6-guard-compiled-out rc=0 ok want=ok as-expected 14.2s +VERIFY29953GUARD COMPLETE 2026-10-05T04:51:06Z all_as_expected=1 diff --git a/docs/maxusai/upstream-mmq-29953-material.md b/docs/maxusai/upstream-mmq-29953-material.md index 1a53e90516c..ea929e28413 100644 --- a/docs/maxusai/upstream-mmq-29953-material.md +++ b/docs/maxusai/upstream-mmq-29953-material.md @@ -39,7 +39,7 @@ the note under "Ten architectures"). Details: "The exact head, measured". `mmq_get_nbytes_shared()` and in its new padding line, spelled differently -- and those two drifting apart is what this bug was. On the stock pool nothing would notice it happen again. "A guard upstream could take" has a patch on the head (+70/-9 lines): one helper for both, and a compile-time check over all ten config tables, verified with -nvcc. Whether to offer it is the maintainer's call. +nvcc and, by the ROCm host, with hipcc. Whether to offer it is the maintainer's call. **For the fork: keep compat 903 until the pin includes #29953**, then retire its padding by the gate under "For the fork" and keep the guard. @@ -253,15 +253,15 @@ ids-branch shapes ([tasks/mmq-successor-results/check-rules-wide.txt](tasks/mmq- | **#29953 head (`3070d927f`)** | **covered** | - | none | > [!NOTE] -> **Corrected 2026-10-05.** This table used to label the published row "#29953 at `3070d927f`". The checker's -> #29953 rule is the published one -- pad by the launched `J_best` in blocks -- and the head's rule was not in it -> at all. Read literally, the old label said upstream's current fix is short in 3.5 million shapes, which is -> false. The head's rule (the launched config's padded tile, and `J_best - 1` for `ids_dst`) is now in -> [tasks/mmq-rules-check.cu](tasks/mmq-rules-check.cu) and is covered in all four tallies on all ten -> architectures: 9,431,816 ids-branch shapes and 203,200 dense ones. Built against `dd266785c` and against -> `3070d927f` itself, the check prints byte-identical output; the config tables are the same files in both. That is by construction -- its padding is -> the load loop's own extent for the launched config -- so the sweep shows the model agrees, not that the head -> is right; the measurements in the Conclusion are the evidence for that. +> **Corrected 2026-10-05.** This table used to label the published row "#29953 at `3070d927f`". The checker's #29953 +> rule is the published one -- pad by the launched `J_best` in blocks -- and the head's rule was not in it at all. +> Read literally, the old label said upstream's current fix is short in 3.5 million shapes, which is false. The +> head's rule (the launched config's padded tile, and `J_best - 1` for `ids_dst`) is now in +> [tasks/mmq-rules-check.cu](tasks/mmq-rules-check.cu) and is covered in all four tallies on all ten architectures: +> 9,431,816 ids-branch shapes and 203,200 dense ones. That is by construction -- its padding is the load loop's own +> extent for the launched config -- so the sweep shows the model agrees, not that the head is right; the +> measurements in the Conclusion are the evidence for that. Built against `dd266785c` and against `3070d927f` +> itself, the check prints byte-identical output; the config tables are the same files in both. **This widens the case for the amendment well beyond gfx1151.** `nthreads` is 512 on CDNA, not 256, so the padded tile is larger: the worst case is CDNA3, q4_0, non-fallback, `J = 64` with `nthreads = 512`, where @@ -596,27 +596,33 @@ would see it happen again: on the stock pool the over-read lands in mapped memor - **A compile-time check.** For every config table and type, the helper must cover the load loop's extent, written from the loop with exact ceiling division -- `ceil(J*MMQ_TILE_Y_K / nthreads) * nthreads` ints -- not from the helper or from `GGML_PAD`. One `static_assert` per (table, type), host pass only: the structure of - the fork's compat 903 guard (#451), which the ROCm host verified under hipcc. + the fork's compat 903 guard (#451). They are gathered by an explicitly instantiated class that inherits every + per-type guard, the ROCm host's variant (#455, #456). The first form gathered them with a `static_assert` on a + fold, which under clang turned non-constant once a per-type assertion failed and added one error per failing + table, worded like a constant-evaluation limit. -Verified with nvcc 12.8 for sm_120 by [tasks/mmq-29953-guard-verify.sh](tasks/mmq-29953-guard-verify.sh), which -scores each build by why it failed -- every error must be the guard's and the tables it names must be the expected +Verified with nvcc 12.8 here and with hipcc on the ROCm host. nvcc, for sm_120, by +[tasks/mmq-29953-guard-verify.sh](tasks/mmq-29953-guard-verify.sh), which scores each build by why it failed -- every error must be the guard's and the tables it names must be the expected ones -- not just by its exit code ([tasks/mmq-successor-results/verify-29953-guard.txt](tasks/mmq-successor-results/verify-29953-guard.txt)): ``` -host load at start (1/5/15 min): 7.73 12.80 17.77, 32 cores -1-head-unpatched rc=0 ok want=ok as-expected 8.1s +host load at start (1/5/15 min): 38.78 35.94 28.60, 32 cores +1-head-unpatched rc=0 ok want=ok as-expected 13.5s patch applies to a pristine 3070d927f6c172242d56a2c1fb897e838e7b2632 -2-head-with-guard rc=0 ok want=ok as-expected 11.2s -3-helper-shrunk-to-J-blocks rc=2 fail want=fail as-expected 3.0s +2-head-with-guard rc=0 ok want=ok as-expected 19.1s +3-helper-shrunk-to-J-blocks rc=2 fail want=fail as-expected 5.0s errors=220, from the guard=220, tables named: ampere blackwell cdna gcn pascal_dp4a pascal_older rdna2 rdna3 rdna3_5 rdna4 -4-helper-rounded-to-one-warp rc=2 fail want=fail as-expected 3.0s +4-helper-rounded-to-one-warp rc=2 fail want=fail as-expected 5.0s errors=220, from the guard=220, tables named: ampere blackwell cdna gcn pascal_dp4a pascal_older rdna2 rdna3 rdna3_5 rdna4 mutation 5 rewrote: CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); -> CASE(GGML_TYPE_MXFP4, 192, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); -5-blackwell-config-at-192-threads rc=2 fail want=fail as-expected 4.0s +5-blackwell-config-at-192-threads rc=2 fail want=fail as-expected 6.1s errors=1, from the guard=1, tables named: blackwell -6-guard-compiled-out rc=0 ok want=ok as-expected 7.9s -VERIFY29953GUARD COMPLETE 2026-10-05T02:33:16Z all_as_expected=1 +mutation 5b rewrote: CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); -> CASE(GGML_TYPE_Q1_0, 160, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); +5b-rdna3_5-config-at-160-threads rc=2 fail want=fail as-expected 6.0s + errors=1, from the guard=1, tables named: rdna3_5 +6-guard-compiled-out rc=0 ok want=ok as-expected 14.2s +VERIFY29953GUARD COMPLETE 2026-10-05T04:51:06Z all_as_expected=1 ``` - **3** puts the published #29953's padding, `J` blocks, back into the helper: every table fails, 220 (table, @@ -626,18 +632,29 @@ VERIFY29953GUARD COMPLETE 2026-10-05T02:33:16Z all_as_expected=1 granularities, the config tables only assert `nthreads % 32 == 0`, and the kernel places `tile_x` with the same `GGML_PAD`. A 192-thread config would compile today and mis-size the tile; with the guard, the build fails and names that table only. -- **Cost:** about 3 s per compile of `mmq.cu`, paid once per build because the guard runs in the host pass only, - however many architectures are targeted: a median of 11.1 s with it against 8.2 s compiled out, over five - interleaved repeats at a load of 7-8 on 32 cores +- **5b** is 5 on an AMD table: rdna3_5 at 160 threads. Not 192: rdna3_5's first 256-thread line has `J = 64`, and + 64 x 36 ints is a multiple of 192, so the bit mask happens to round correctly there. The first instruction to + the ROCm host said 192 and could not fire; they found 160 (#455). +- **hipcc agrees on every build** (ROCm 7.2.1, AMD clang 22, gfx1151, run by the ROCm host: + [#456](https://github.com/MaxusAI/ollama/pull/456), `rocm-gfx1151/README.md`). The patch compiles; 3 and 4 fail + with 220 errors, all the guard's, naming all ten tables; blackwell at 192 and rdna3_5 at 160 each name only + their table; compiled out, it compiles. clang stops at 20 errors unless given `-ferror-limit=0`. +- **No generated code changes, on either compiler.** On hipcc `mmq.o` is byte-identical with and without the + guard: clang derives `__hip_cuid_*` from the path and the command line. On nvcc whole objects differ between + two builds of the same source, so the comparison is of the host disassembly, 56,837 lines, identical across the + first form, the variant, no guard and a repeat build. +- **Cost:** about 3 s per compile of `mmq.cu` on nvcc and 0.8 s on hipcc, paid once per build because the guard + runs in the host pass only, however many architectures are targeted. nvcc: a median of 15.2 s with the guard + against 11.8 s compiled out, over five interleaved repeats at a load of 28-36 on 32 cores ([tasks/mmq-29953-guard-timing.sh](tasks/mmq-29953-guard-timing.sh), - [output](tasks/mmq-successor-results/guard-timing.txt)). A single build is too noisy to quote: two one-off - runs gave 9.8 against 8.9 s and 11.2 against 7.9 s. + [output](tasks/mmq-successor-results/guard-timing.txt)); the first form measured 11.1 against 8.2 s at a load + of 7-8. hipcc, measured by the ROCm host: 2.89 against 2.06 s, median of five. - nvcc's front end stops at 100 errors by default, which on the first run cut mutation 3's list to the first five tables; the script passes `-Xcudafe --error_limit=100000` so every failing table is named. -**Limits.** Only nvcc was run here: the HIP and MUSA branches are untested on this host. HIP keys off -`__HIP_DEVICE_COMPILE__` because `vendors/hip.h` defines `__CUDA_ARCH__` in every HIP pass -- the trap the ROCm -host caught in #451's first draft -- and MUSA is excluded, as in 903, because nothing here builds it. The guard +**Limits.** The HIP branch is verified by the ROCm host; MUSA is untested, and excluded as in 903 because nothing +here builds it. HIP keys off `__HIP_DEVICE_COMPILE__` because `vendors/hip.h` defines `__CUDA_ARCH__` in every HIP +pass -- the trap the ROCm host caught in #451's first draft. The guard cannot see call sites: a later change that pads src1 by something other than the helper would pass it. `ids_dst` needs no guard of this kind; its `J_best - 1` depends on nothing but `J`. diff --git a/llama/compat/README.md b/llama/compat/README.md index 3989d88a8c8..87d5160efa4 100644 --- a/llama/compat/README.md +++ b/llama/compat/README.md @@ -217,7 +217,7 @@ intentionally skipped so a developer can iterate on a local llama.cpp tree. guard-only, one helper for the y tile and a `static_assert` against the load loop, so that a later upstream change that shrinks the padding fails the build here. `docs/maxusai/tasks/mmq-29953-y-tile-guard.patch` is that - form on #29953's head, verified with nvcc. + form on #29953's head, verified with nvcc and hipcc. - Check the series on a checkout before building. - **References.** - `docs/maxusai/qwen35moe-mmq-investigation.md`: the original diagnosis.