From 02a2d6ef124b1d483ec643a793d18d381b5bd981 Mon Sep 17 00:00:00 2001 From: Nikolay Nadorichev Date: Wed, 28 Oct 2026 22:48:51 +0300 Subject: [PATCH] Broadcast subgroupShuffle with a uniform index instead of looping (linux/0006) ir3 lowers shuffle(x, i) to rotate(x, i - gl_SubgroupInvocationID) in a loop with one iteration per distinct relative offset, so a broadcast costs about one iteration per invocation; with the half-wave subgroups patch the index is never uniform and every such shuffle runs ~32 iterations. linux/0006 lowers a uniform index to read_invocation_cond_ir3 and, for a divergent one, serves the invocations with the same absolute source and the ones with the same relative offset in the same iteration. In 007 First Light on Adreno 840 one compute shader went from 94.6 to 8.4 ms per frame. --- README.md | 2 +- .../linux/0006-ir3-shuffle-broadcast.patch | 98 +++++++++++++++++++ patches/series.json | 7 ++ 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 patches/linux/0006-ir3-shuffle-broadcast.patch diff --git a/README.md b/README.md index d609dfa..56e529a 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ build and the release notes. | `scripts/android_ndk_compat.py` | ✓ | | | `scripts/linux_kgsl_wsi.py` | | ✓ | | `scripts/process_heap_usage.py` | ✓ | ✓ | -| `linux/0001` … `linux/0005` (mesh shaders, half-wave subgroups, A8XX cube, bindless and IB fixes) | | ✓ | +| `linux/0001` … `linux/0006` (mesh shaders, half-wave subgroups, A8XX cube, bindless and IB fixes, subgroup shuffle broadcast) | | ✓ | | `lrz/0001` (A8XX LRZ flush before the next render pass) | ✓ | ✓ | A script whose anchor no longer matches Mesa main exits non-zero and a `.patch` that no longer applies fails diff --git a/patches/linux/0006-ir3-shuffle-broadcast.patch b/patches/linux/0006-ir3-shuffle-broadcast.patch new file mode 100644 index 0000000..34da611 --- /dev/null +++ b/patches/linux/0006-ir3-shuffle-broadcast.patch @@ -0,0 +1,98 @@ +diff --git a/src/freedreno/ir3/ir3_lower_subgroups.c b/src/freedreno/ir3/ir3_lower_subgroups.c +index c95780a..2a25cce 100644 +--- a/src/freedreno/ir3/ir3_lower_subgroups.c ++++ b/src/freedreno/ir3/ir3_lower_subgroups.c +@@ -838,6 +838,67 @@ make_shuffle_uniform(nir_builder *b, nir_def *val, nir_def *index, + return nir_load_var(b, result); + } + ++/* Variant of make_shuffle_uniform for shuffle(val, abs_index) with a divergent ++ * abs_index. Each iteration takes the absolute index and the relative offset ++ * of the first unserved invocation, and serves every invocation that wants the ++ * same absolute source (a broadcast through read_invocation_cond_ir3) as well ++ * as every invocation that wants the same relative offset (a uniform rotate). ++ * Each iteration retires all invocations of at least one distinct absolute ++ * index and one distinct relative offset, so the loop never takes more ++ * iterations than make_shuffle_uniform, and a broadcast with a divergent index ++ * (e.g. one source per half of the wave) takes one iteration per distinct ++ * source instead of roughly one per invocation. An iteration costs a few more ++ * instructions than in make_shuffle_uniform. All invocations stay active for ++ * the shfl, as in make_shuffle_uniform. ++ */ ++static nir_def * ++make_shuffle_mixed(nir_builder *b, nir_def *val, nir_def *abs_index, ++ nir_def *rel_index) ++{ ++ nir_variable *done = ++ nir_local_variable_create(b->impl, glsl_bool_type(), "done"); ++ nir_store_var(b, done, nir_imm_false(b), 1); ++ nir_variable *result = ++ nir_local_variable_create(b->impl, glsl_type_for_def(val), "result"); ++ nir_store_var(b, result, nir_undef(b, 1, val->bit_size), 1); ++ ++ nir_loop *loop = nir_push_loop(b); ++ { ++ nir_def *was_done = nir_load_var(b, done); ++ nir_def *pending = nir_inot(b, was_done); ++ nir_def *next_abs = nir_read_invocation_cond_ir3( ++ b, abs_index->bit_size, abs_index, pending); ++ next_abs->divergent = false; ++ nir_def *next_rel = nir_read_invocation_cond_ir3( ++ b, rel_index->bit_size, rel_index, pending); ++ next_rel->divergent = false; ++ ++ nir_def *bcast = nir_read_invocation_cond_ir3( ++ b, val->bit_size, val, ++ nir_ieq(b, nir_load_subgroup_invocation(b), next_abs)); ++ bcast->divergent = false; ++ nir_def *rot = shuffle_to_uniform(b, nir_intrinsic_shuffle, val, next_rel); ++ ++ nir_def *hit_abs = nir_ieq(b, abs_index, next_abs); ++ nir_def *hit_rel = nir_ieq(b, rel_index, next_rel); ++ nir_def *take_abs = nir_iand(b, pending, hit_abs); ++ nir_def *take_rel = ++ nir_iand(b, pending, nir_iand(b, nir_inot(b, hit_abs), hit_rel)); ++ ++ nir_def *res = nir_load_var(b, result); ++ res = nir_bcsel(b, take_rel, rot, res); ++ res = nir_bcsel(b, take_abs, bcast, res); ++ nir_store_var(b, result, res, 1); ++ ++ nir_def *now_done = nir_ior(b, was_done, nir_ior(b, hit_abs, hit_rel)); ++ nir_store_var(b, done, now_done, 1); ++ nir_break_if(b, nir_vote_all(b, 1, now_done)); ++ } ++ nir_pop_loop(b, loop); ++ ++ return nir_load_var(b, result); ++} ++ + static nir_def * + lower_shuffle(nir_builder *b, nir_instr *instr, void *data) + { +@@ -846,11 +907,25 @@ lower_shuffle(nir_builder *b, nir_instr *instr, void *data) + nir_def *index = intrin->src[1].ssa; + + if (intrin->intrinsic == nir_intrinsic_shuffle) { ++ nir_def *abs_index = index; ++ /* A uniform absolute index is a broadcast, which needs no loop. The new ++ * paths are only used for 32-bit values; other bit sizes keep the ++ * rotate-based lowering below. ++ */ ++ if (!abs_index->divergent && val->bit_size == 32) { ++ nir_def *res = nir_read_invocation_cond_ir3( ++ b, val->bit_size, val, ++ nir_ieq(b, nir_load_subgroup_invocation(b), abs_index)); ++ res->divergent = false; ++ return res; ++ } + /* The hw only does relative shuffles/rotates so transform shuffle(val, x) + * into rotate(val, x - gl_SubgroupInvocationID) which is valid since we + * make sure to only use it with uniform indices. + */ + index = nir_isub(b, index, nir_load_subgroup_invocation(b)); ++ if (val->bit_size == 32) ++ return make_shuffle_mixed(b, val, abs_index, index); + } + + if (!index->divergent) { diff --git a/patches/series.json b/patches/series.json index 4047232..04cc457 100644 --- a/patches/series.json +++ b/patches/series.json @@ -156,5 +156,12 @@ "linux" ], "summary": "Reset the LRZ controls after the end-of-pass LRZ flush on A8XX, as freedreno does, so a large fast-cleared LRZ buffer no longer hangs the GPU" + }, + { + "file": "linux/0006-ir3-shuffle-broadcast.patch", + "targets": [ + "linux" + ], + "summary": "Lower subgroupShuffle with a uniform index to a broadcast and serve equal source indices together, instead of a loop over every relative offset" } ]