From d1f04b3c4c0f503416fdf70b05c72496a8c24968 Mon Sep 17 00:00:00 2001 From: Stephen Meehan Date: Mon, 9 Nov 2026 23:03:25 +0100 Subject: [PATCH 1/6] shader: implement Ampere F2IP byte packing --- .../Instructions/InstEmitAmpereAlu.cs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereAlu.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereAlu.cs index 8b2b57f6..b181718d 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereAlu.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereAlu.cs @@ -35,6 +35,7 @@ public static void RegisterAmpereAlu(AmpereAluRegistrar alu, AmpereRegistrar x) alu(0x107, InstName.F2f, AFrnd); alu(0x104, InstName.F2f, AF2f); alu(0x03e, InstName.F2f, AF2fp); + alu(0x043, InstName.F2i, AF2ip); alu(0x030, InstName.Hadd2, AHadd2); alu(0x032, InstName.Hmul2, AHmul2); alu(0x031, InstName.Hfma2, AHfma2); @@ -433,6 +434,36 @@ public static void AF2fp(EmitterContext context) ASet(context, x, context.PackHalf2x16(AFloat(context, b), AFloat(context, a)), AUniform(context)); } + // F2IP.U8 Rd, a, b: a and b to saturated bytes, a in byte 1 and b in + // byte 0, like F2FP's halves; bits 78..79 select F2I rounding. + public static void AF2ip(EmitterContext context) + { + AmpereOp x = new(context.CurrOp); + + if (x.Bits(72, 78) != 4 || x.Rc != RegisterConsts.RegisterZeroIndex) + { + throw AFail(context, "F2IP form"); + } + + (ASrc a, ASrc b) = ASecond(context, x, abs: false); + RoundMode2 mode = (RoundMode2)x.Bits(78, 80); + + Operand Byte(Operand f) + { + f = mode switch + { + RoundMode2.Round => context.FPRound(f, Instruction.FP32), + RoundMode2.Floor => context.FPFloor(f, Instruction.FP32), + RoundMode2.Ceil => context.FPCeiling(f, Instruction.FP32), + _ => context.FPTruncate(f, Instruction.FP32), + }; + + return context.FP32ConvertToU32(context.FPMinimum(context.FPMaximum(f, ConstF(0f)), ConstF(255f))); + } + + ASet(context, x, context.BitwiseOr(context.ShiftLeft(Byte(AFloat(context, a)), Const(8)), Byte(AFloat(context, b))), AUniform(context)); + } + // ---- packed halves ---- // A half-pair operand's two lanes after its swizzle (0 as is, 2 low twice, 3 high twice; 1 the high half of a scalar), abs / neg applied. From 68caf7517c847d34cc1c4ab5c336073c3cf174ab Mon Sep 17 00:00:00 2001 From: seedds Date: Wed, 11 Nov 2026 19:35:55 +0100 Subject: [PATCH 2/6] shader: let tasks read back their payload Ampere opcode 0x923 mode 0xb800 reads the task payload written through opcode 0x927 mode 0xb800. Keep this view distinct from the mesh stage's task-input view and cover it with a hand-encoded TaskEXT fixture whose loaded word controls OpEmitMeshTasksEXT. --- dm/gfx/tests/test_dmsh_ampere_mesh.py | 8 ++++++++ .../Instructions/InstEmitAmpereMesh.cs | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/dm/gfx/tests/test_dmsh_ampere_mesh.py b/dm/gfx/tests/test_dmsh_ampere_mesh.py index 80a1f49b..a668cfc0 100644 --- a/dm/gfx/tests/test_dmsh_ampere_mesh.py +++ b/dm/gfx/tests/test_dmsh_ampere_mesh.py @@ -86,6 +86,14 @@ def test_task(self): self.assertEqual(ops[OP_EMIT_MESH_TASKS][0][3], payload[0][1]) # the payload goes with the launch self.assertEqual(res['stage'], 'ts') + def test_task_reads_its_own_payload(self): + # 0x927 / 0xb8 writes a record, then 0x923 / 0xb8 reads the same word in another thread. The loaded value becomes the mesh-workgroup count, so it survives optimization. + code = program(stage(STAGE_LOAD, HEADER, rd=0), stage(STAGE_STORE, TASK_WRITE, rb=0, imm=0x10), + stage(STAGE_LOAD, TASK_WRITE, rd=1, imm=0x10), stage(STAGE_STORE, TASK_WRITE, rb=1, imm=4)) + (_, ops), _ = self.translate(code, stage='ts', local=[32, 1, 1], mesh=[0, 0, 64]) + self.assertEqual([e[0] for e in ops[OP_ENTRY_POINT]], [MODEL_TASK]) + self.assertEqual(len(ops[OP_EMIT_MESH_TASKS]), 1) + # A mesh program: its index (task memory word 0), a payload word by it, the four position planes and the generic planes of vertex 0, one per-primitive value of # primitive 0, triangle 0's indices (bytes 4..6) and the primitive count (byte 3) MESH = program(stage(STAGE_LOAD, TASK_READ, rd=1), stage(STAGE_LOAD, TASK_READ, rd=2, ra=1, imm=0x10), diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMesh.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMesh.cs index 5a575444..488c27ee 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMesh.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMesh.cs @@ -45,6 +45,7 @@ private enum AStageWindow None, Header, TaskMemoryRead, + TaskMemoryOwnRead, TaskMemoryWrite, VertexPlanes, PrimitiveIndices, @@ -82,7 +83,7 @@ public static void AStageMemory(EmitterContext context) int offset = x.Bits(40, 64); AStageWindow window = load - ? mode switch { 0x7800 => AStageWindow.Header, 0x3800 => AStageWindow.TaskMemoryRead, _ => AStageWindow.None } + ? mode switch { 0x7800 => AStageWindow.Header, 0x3800 => AStageWindow.TaskMemoryRead, 0xb800 => AStageWindow.TaskMemoryOwnRead, _ => AStageWindow.None } : mode switch { 0xb800 => AStageWindow.TaskMemoryWrite, 0xf800 => AStageWindow.VertexPlanes, 0x8000 => AStageWindow.PrimitiveIndices, 0x8800 => AStageWindow.PrimitiveHeader, _ => AStageWindow.None }; if (window == AStageWindow.None) @@ -95,7 +96,7 @@ public static void AStageMemory(EmitterContext context) throw AFail(context, "stage memory form"); } - bool taskWindow = window is AStageWindow.TaskMemoryWrite; + bool taskWindow = window is AStageWindow.TaskMemoryOwnRead or AStageWindow.TaskMemoryWrite; bool meshWindow = window is AStageWindow.TaskMemoryRead or AStageWindow.VertexPlanes or AStageWindow.PrimitiveIndices or AStageWindow.PrimitiveHeader; if ((taskWindow && !mesh.IsTask) || (meshWindow && !mesh.IsMesh)) @@ -142,6 +143,17 @@ public static void AStageMemory(EmitterContext context) ASet(context, x, x.Ra == RegisterConsts.RegisterZeroIndex ? loaded : context.ConditionalSelect(context.ICompareEqual(word, Const(0)), index, loaded)); } + return; + + case AStageWindow.TaskMemoryOwnRead: + // A task may read back the payload it wrote through 0x927 / 0xb800. Ocarina of Time's 5d3a096025e375a1 writes records in one thread and + // consumes them in another; this is the same task payload memory, not the mesh workgroup's TaskMemoryRead view. + if (x.Rd != RegisterConsts.RegisterZeroIndex) + { + Operand word = context.ShiftRightU32(context.BitwiseAnd(address, Const(mesh.PayloadWords * 4 - 1)), Const(2)); + ASet(context, x, context.Load(StorageKind.SharedMemory, mesh.PayloadMemoryId, word)); + } + return; } From 9ec4050b585d474701fcc7d9119f00c33aa119a6 Mon Sep 17 00:00:00 2001 From: Stephen Meehan Date: Wed, 11 Nov 2026 19:49:46 +0100 Subject: [PATCH 3/6] shader: translate point and line mesh topology --- dm/gfx/tests/test_dmsh_ampere_mesh.py | 28 ++++++- .../CodeGen/Spirv/SpirvMeshEntry.cs | 74 ++++++++++++++----- .../StructuredIr/MeshShaderInfo.cs | 31 ++++++-- .../ryujinx/src/Spv.Generator/spirv.cs | 3 + 4 files changed, 109 insertions(+), 27 deletions(-) diff --git a/dm/gfx/tests/test_dmsh_ampere_mesh.py b/dm/gfx/tests/test_dmsh_ampere_mesh.py index a668cfc0..3fd98009 100644 --- a/dm/gfx/tests/test_dmsh_ampere_mesh.py +++ b/dm/gfx/tests/test_dmsh_ampere_mesh.py @@ -28,9 +28,9 @@ def program(*insts): return bytes(0x80) + b''.join(insts) + inst(*EXIT) # a ze # SPIR-V numbers the checks read OP_ENTRY_POINT, OP_EXECUTION_MODE, OP_DECORATE, OP_VARIABLE, OP_EMIT_MESH_TASKS, OP_SET_MESH_OUTPUTS = 15, 16, 71, 59, 5294, 5295 MODEL_TASK, MODEL_MESH = 5364, 5365 -MODE_LOCAL_SIZE, MODE_OUTPUT_VERTICES, MODE_OUTPUT_PRIMITIVES, MODE_OUTPUT_TRIANGLES = 17, 26, 5270, 5298 +MODE_LOCAL_SIZE, MODE_OUTPUT_POINTS, MODE_OUTPUT_VERTICES, MODE_OUTPUT_PRIMITIVES, MODE_OUTPUT_LINES, MODE_OUTPUT_TRIANGLES = 17, 27, 26, 5270, 5269, 5298 DEC_SPEC_ID, DEC_BUILTIN, DEC_LOCATION, DEC_PER_PRIMITIVE = 1, 11, 30, 5271 -BUILTIN_PRIMITIVE_TRIANGLE_INDICES = 5296 +BUILTIN_POINT_SIZE, BUILTIN_PRIMITIVE_POINT_INDICES, BUILTIN_PRIMITIVE_LINE_INDICES, BUILTIN_PRIMITIVE_TRIANGLE_INDICES = 1, 5294, 5295, 5296 SC_TASK_PAYLOAD = 5402 @@ -94,6 +94,9 @@ def test_task_reads_its_own_payload(self): self.assertEqual([e[0] for e in ops[OP_ENTRY_POINT]], [MODEL_TASK]) self.assertEqual(len(ops[OP_EMIT_MESH_TASKS]), 1) + def test_task_refuses_mesh_topology(self): + self.refused(self.TASK, 'task program with mesh output', stage='ts', local=[32, 1, 1], mesh=[0, 0, 64, -5, 1]) + # A mesh program: its index (task memory word 0), a payload word by it, the four position planes and the generic planes of vertex 0, one per-primitive value of # primitive 0, triangle 0's indices (bytes 4..6) and the primitive count (byte 3) MESH = program(stage(STAGE_LOAD, TASK_READ, rd=1), stage(STAGE_LOAD, TASK_READ, rd=2, ra=1, imm=0x10), @@ -129,6 +132,27 @@ def test_mesh_without_payload(self): # a mesh program run without a task reads stage='ms', local=[32, 1, 1], mesh=[32, 16, 128, 0, 1, 2, 3]) self.assertFalse([v for v in ops[OP_VARIABLE] if v[2] == SC_TASK_PAYLOAD]) + def test_point_mesh_with_point_size(self): + code = program(*[stage(STAGE_STORE, PLANES, rb=1, imm=0x80 * p) for p in range(5)], + stage(STAGE_STORE, INDICES, rb=RZ, imm=4), stage(STAGE_STORE, COUNTS, rb=1, imm=3)) + (_, ops), _ = self.translate(code, stage='ms', local=[32, 1, 1], mesh=[32, 32, 64, -5, 1, -4, 0, 1, 2, 3]) + ep = ops[OP_ENTRY_POINT][0] + modes = {m[1]: m[2:] for m in ops[OP_EXECUTION_MODE] if m[0] == ep[1]} + self.assertIn(MODE_OUTPUT_POINTS, modes) + decos = [d for d in ops[OP_DECORATE] if d[1] == DEC_BUILTIN] + self.assertTrue(any(d[2] == BUILTIN_PRIMITIVE_POINT_INDICES for d in decos)) + self.assertTrue(any(d[2] == DEC_BUILTIN and d[3] == BUILTIN_POINT_SIZE for d in ops.get(72, []))) + + def test_line_mesh(self): + code = program(*[stage(STAGE_STORE, PLANES, rb=1, imm=0x80 * p) for p in range(4)], + stage(STAGE_STORE, INDICES, rb=RZ, imm=4), stage(STAGE_STORE, INDICES, rb=1, imm=5), + stage(STAGE_STORE, COUNTS, rb=1, imm=3)) + (_, ops), _ = self.translate(code, stage='ms', local=[32, 1, 1], mesh=[32, 32, 64, -5, 2, 0, 1, 2, 3]) + ep = ops[OP_ENTRY_POINT][0] + modes = {m[1]: m[2:] for m in ops[OP_EXECUTION_MODE] if m[0] == ep[1]} + self.assertIn(MODE_OUTPUT_LINES, modes) + self.assertTrue(any(d[1] == DEC_BUILTIN and d[2] == BUILTIN_PRIMITIVE_LINE_INDICES for d in ops[OP_DECORATE])) + def test_refused_mode(self): self.refused(program(stage(STAGE_STORE, 0x9800, rb=0)), 'stage memory mode 0x9800', stage='ms', local=[32, 1, 1], mesh=[32, 16, 128, 0, 1, 2, 3]) diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvMeshEntry.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvMeshEntry.cs index c74ffd60..b82112b0 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvMeshEntry.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvMeshEntry.cs @@ -15,11 +15,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv // task (TaskEXT): before the body the task memory's word 1 (mesh workgroups to launch) is 0; after it OpEmitMeshTasksEXT(word 1, 1, 1, payload) ends every // invocation (switch2mac also cleared word 0, which a mesh workgroup reads as its own index: InstEmitAmpereMesh.cs) // mesh (MeshEXT): OpSetMeshOutputsEXT(max vertices, max primitives) first (no output is written before it), then the outputs that start as (0, 0, 0, 1) set, - // the body (its plane stores land in the outputs at once, MeshPlaneStore below), then every triangle's indices from the primitive area: the program's - // indices (each clamped below max vertices) for t below the primitive count (byte 3), (0, 0, 0), a degenerate triangle, from there to max primitives; + // the body (its plane stores land in the outputs at once, MeshPlaneStore below), then every point, line or triangle's indices from the primitive area: the program's + // indices (each clamped below max vertices) for t below the primitive count (byte 3), zero indices for degenerate unused primitives from there to max primitives; // with a GL depth constant (MeshShaderInfo.GlDepthSpecId), z' = z / 2 + w / 2 of every vertex when it is true, as vk_android_spv.gl_depth does a vertex shader's - // Outputs: the position in a per-vertex block array (BuiltIn Position), one vec4 array per per-vertex generic location at Location g (as this translator's vertex - // shaders number theirs), one vec4 array per per-primitive location at Location g decorated PerPrimitiveEXT, PrimitiveTriangleIndicesEXT uvec3[max primitives]. + // Outputs: the point size and position in a per-vertex block array (BuiltIn PointSize / Position), one vec4 array per per-vertex generic location at Location g (as this translator's vertex + // shaders number theirs), one vec4 array per per-primitive location at Location g decorated PerPrimitiveEXT, and the topology's Primitive*IndicesEXT array. // SPIR-V 1.4: the entry point lists every global variable. // Dropped from switch2mac's Metal path: the per-vertex outputs at every location 0-5 whether written or not (Metal refused a mesh pipeline missing a fragment // input; here only the layout's padded locations are added), the 1024 mesh workgroup bound of a Metal object function (here 65535, Vulkan's @@ -36,6 +36,7 @@ public static void Generate(CodeGenContext context, MeshShaderInfo mesh) SpvInstruction tVoid = context.TypeVoid(), tU32 = context.TypeU32(), tF32 = context.TypeFP32(), tBool = context.TypeBool(); SpvInstruction tVec4 = context.TypeVector(tF32, 4), tUVec3 = context.TypeVector(tU32, 3); + SpvInstruction tUVec2 = mesh.IsMesh && mesh.PrimitiveVertices == 2 ? context.TypeVector(tU32, 2) : null; SpvInstruction U(uint v) => context.Constant(tU32, v); uint threads = (uint)Math.Max(1, context.Definitions.ComputeLocalSizeX * context.Definitions.ComputeLocalSizeY * context.Definitions.ComputeLocalSizeZ); List io = new(context.GetMainInterface()); @@ -55,9 +56,14 @@ public static void Generate(CodeGenContext context, MeshShaderInfo mesh) if (mesh.IsMesh) { - SpvInstruction perVertex = context.TypeStruct(true, tVec4); + bool pointSize = mesh.Planes.Contains(MeshShaderInfo.PointSizePlane); + SpvInstruction perVertex = pointSize ? context.TypeStruct(true, tVec4, tF32) : context.TypeStruct(true, tVec4); context.Decorate(perVertex, Decoration.Block); context.MemberDecorate(perVertex, 0, Decoration.BuiltIn, (SpvLiteralInteger)BuiltIn.Position); + if (pointSize) + { + context.MemberDecorate(perVertex, 1, Decoration.BuiltIn, (SpvLiteralInteger)BuiltIn.PointSize); + } posVar = context.Variable(context.TypePointer(StorageClass.Output, context.TypeArray(perVertex, U((uint)mesh.MaxVertices))), StorageClass.Output); context.Name(posVar, "mesh_position"); context.AddGlobalVariable(posVar); @@ -84,8 +90,15 @@ public static void Generate(CodeGenContext context, MeshShaderInfo mesh) io.Add(v); } - idxVar = context.Variable(context.TypePointer(StorageClass.Output, context.TypeArray(tUVec3, U((uint)mesh.MaxPrimitives))), StorageClass.Output); - context.Decorate(idxVar, Decoration.BuiltIn, (SpvLiteralInteger)BuiltIn.PrimitiveTriangleIndicesEXT); + SpvInstruction indexType = mesh.PrimitiveVertices switch { 1 => tU32, 2 => tUVec2, _ => tUVec3 }; + BuiltIn indexBuiltIn = mesh.PrimitiveVertices switch + { + 1 => BuiltIn.PrimitivePointIndicesEXT, + 2 => BuiltIn.PrimitiveLineIndicesEXT, + _ => BuiltIn.PrimitiveTriangleIndicesEXT, + }; + idxVar = context.Variable(context.TypePointer(StorageClass.Output, context.TypeArray(indexType, U((uint)mesh.MaxPrimitives))), StorageClass.Output); + context.Decorate(idxVar, Decoration.BuiltIn, (SpvLiteralInteger)indexBuiltIn); context.AddGlobalVariable(idxVar); io.Add(idxVar); } @@ -163,7 +176,8 @@ void ForEach(SpvInstruction lidValue, int count, Action then) // position and the generic locations whose components the planes do not all name, and every per-primitive location bool posFull = Enumerable.Range(0, 4).All(c => mesh.Planes.Contains(c)); List vertexInit = generics.Where(kv => !Enumerable.Range(0, 4).All(c => mesh.Planes.Contains(4 + 4 * kv.Key + c))).Select(kv => kv.Value).ToList(); - if (vertexInit.Count != 0 || !posFull) + bool pointSize = mesh.Planes.Contains(MeshShaderInfo.PointSizePlane); + if (vertexInit.Count != 0 || !posFull || pointSize) { ForEach(lid, mesh.MaxVertices, v => { @@ -172,6 +186,11 @@ void ForEach(SpvInstruction lidValue, int count, Action then) context.Store(context.AccessChain(pOutVec4, posVar, v, U(0)), init); } + if (pointSize) + { + context.Store(context.AccessChain(pOutF32, posVar, v, U(1)), context.Constant(tF32, 1f)); + } + foreach (SpvInstruction g in vertexInit) { context.Store(context.AccessChain(pOutVec4, g, v), init); @@ -215,7 +234,8 @@ void ForEach(SpvInstruction lidValue, int count, Action then) { SpvInstruction prims = context.SharedMemories[mesh.PrimitiveMemoryId]; SpvInstruction primCount = context.Load(tU32, context.AccessChain(pWg, prims, U(3))); - SpvInstruction pOutUVec3 = context.TypePointer(StorageClass.Output, tUVec3); + SpvInstruction indexType = mesh.PrimitiveVertices switch { 1 => tU32, 2 => tUVec2, _ => tUVec3 }; + SpvInstruction pOutIndex = context.TypePointer(StorageClass.Output, indexType); SpvInstruction lastVertex = U((uint)mesh.MaxVertices - 1); SpvInstruction Index(SpvInstruction word) @@ -227,10 +247,23 @@ SpvInstruction Index(SpvInstruction word) ForEach(lid, mesh.MaxPrimitives, t => { - SpvInstruction a = context.IAdd(tU32, context.IMul(tU32, t, U(3)), U(4)); - SpvInstruction tri = context.CompositeConstruct(tUVec3, Index(a), Index(context.IAdd(tU32, a, U(1))), Index(context.IAdd(tU32, a, U(2)))); - tri = context.Select(tUVec3, context.ULessThan(tBool, t, primCount), tri, context.ConstantComposite(tUVec3, U(0), U(0), U(0))); - context.Store(context.AccessChain(pOutUVec3, idxVar, t), tri); + if (mesh.PrimitiveVertices == 3) + { + SpvInstruction a = context.IAdd(tU32, context.IMul(tU32, t, U(3)), U(4)); + SpvInstruction tri = context.CompositeConstruct(tUVec3, Index(a), Index(context.IAdd(tU32, a, U(1))), Index(context.IAdd(tU32, a, U(2)))); + tri = context.Select(tUVec3, context.ULessThan(tBool, t, primCount), tri, context.ConstantComposite(tUVec3, U(0), U(0), U(0))); + context.Store(context.AccessChain(pOutIndex, idxVar, t), tri); + } + else + { + SpvInstruction a = context.IAdd(tU32, context.IMul(tU32, t, U((uint)mesh.PrimitiveVertices)), U(4)); + SpvInstruction indices = mesh.PrimitiveVertices == 1 + ? Index(a) + : context.CompositeConstruct(tUVec2, Index(a), Index(context.IAdd(tU32, a, U(1)))); + SpvInstruction zero = mesh.PrimitiveVertices == 1 ? U(0) : context.ConstantComposite(tUVec2, U(0), U(0)); + indices = context.Select(indexType, context.ULessThan(tBool, t, primCount), indices, zero); + context.Store(context.AccessChain(pOutIndex, idxVar, t), indices); + } }); // The GL depth convention (vk_android_spv.gl_depth's arithmetic for a vertex shader): z' = fma(z, 0.5, w * 0.5) when the constant is true @@ -285,9 +318,11 @@ void Scatter(SpvInstruction w, int chunkWords, int count, int[] planes, Action Scatter(word, mesh.VertexChunkWords, mesh.MaxVertices, mesh.Planes, (v, plane) => { - context.Store(plane < 4 - ? context.AccessChain(pOutF32, posVar, v, U(0), U((uint)plane)) - : context.AccessChain(pOutF32, generics[(plane - 4) >> 2], v, U((uint)(plane & 3))), f); + context.Store(plane == MeshShaderInfo.PointSizePlane + ? context.AccessChain(pOutF32, posVar, v, U(1)) + : plane < 4 + ? context.AccessChain(pOutF32, posVar, v, U(0), U((uint)plane)) + : context.AccessChain(pOutF32, generics[(plane - 4) >> 2], v, U((uint)(plane & 3))), f); if (zw != null && plane is 2 or 3) { @@ -326,7 +361,12 @@ void Scatter(SpvInstruction w, int chunkWords, int count, int[] planes, Action ExecutionMode.OutputPoints, + 2 => ExecutionMode.OutputLinesEXT, + _ => ExecutionMode.OutputTrianglesEXT, + }); } } } diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MeshShaderInfo.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MeshShaderInfo.cs index b83c875c..1b911f6c 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MeshShaderInfo.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MeshShaderInfo.cs @@ -16,8 +16,9 @@ namespace Ryujinx.Graphics.Shader.StructuredIr // (t % 32) x 4). Its stores go straight into the mesh outputs (CodeGenContext.MeshPlaneStore); the memory itself is one word nothing reads // the primitive area, mesh: byte 3 the primitive count, 4 + 3 t ... triangle t's three vertex indices. One word per byte (each store a whole word, so no // byte merging between invocations), PrimitiveWords words, addresses masked - // The layout (IGpuAccessor.QueryMeshShaderLayout, the job's "mesh"): max vertices, max primitives, payload bytes, the per-vertex plane codes in the order of the - // planes (the SPH's output map: 0-3 position x y z w, 4 + 4 g + c generic g component c), then optionally -1 and the per-primitive value codes (same numbering, + // The layout (IGpuAccessor.QueryMeshShaderLayout, the job's "mesh"): max vertices, max primitives, payload bytes, optionally -5 and the primitive's vertex count, + // then the per-vertex plane codes in the order of the planes (the SPH's output map: -4 point size, 0-3 position x y z w, 4 + 4 g + c generic g component c), + // then optionally -1 and the per-primitive value codes (same numbering, // generic only; -3 for a value the program writes that no output takes: the count sets the chunk stride, values x 0x80), then optionally -2 and the generic locations to declare as per-vertex outputs although the program writes none of their components (the // pixel shader reads them: they hold (0, 0, 0, 1)) class MeshShaderInfo @@ -26,6 +27,8 @@ class MeshShaderInfo // A per-primitive value no output takes (its stores are dropped) public const int NoOutput = -3; + public const int PointSizePlane = -4; + public const int Topology = -5; public bool IsTask { get; } public bool IsMesh => !IsTask; @@ -33,6 +36,7 @@ class MeshShaderInfo public int MaxPrimitives { get; } public int PayloadBytes { get; } public int PayloadWords { get; } + public int PrimitiveVertices { get; } public int[] Planes { get; } public int[] PrimitivePlanes { get; } public int[] PadLocations { get; } @@ -59,13 +63,23 @@ public MeshShaderInfo(bool isTask, int[] layout) MaxPrimitives = layout[1]; PayloadBytes = layout[2]; - int primSplit = Array.IndexOf(layout, -1, 3); - int padSplit = Array.IndexOf(layout, -2, 3); + int planeStart = 3; + PrimitiveVertices = 3; + + bool hasTopology = layout.Length >= 5 && layout[3] == Topology; + if (hasTopology) + { + PrimitiveVertices = layout[4]; + planeStart = 5; + } + + int primSplit = Array.IndexOf(layout, -1, planeStart); + int padSplit = Array.IndexOf(layout, -2, planeStart); int end = layout.Length; int planeEnd = primSplit >= 0 ? primSplit : padSplit >= 0 ? padSplit : end; int primEnd = padSplit >= 0 ? padSplit : end; - Planes = layout[3..planeEnd]; + Planes = layout[planeStart..planeEnd]; PrimitivePlanes = primSplit >= 0 ? layout[(primSplit + 1)..primEnd] : Array.Empty(); PadLocations = padSplit >= 0 ? layout[(padSplit + 1)..end] : Array.Empty(); @@ -76,7 +90,8 @@ public MeshShaderInfo(bool isTask, int[] layout) throw new NotSupportedException($"mesh program with {MaxVertices} vertices / {MaxPrimitives} primitives at most (1-256 each)"); } - if (Planes.Any(p => p < 0 || p >= 4 + 4 * 32) || PrimitivePlanes.Any(p => p != NoOutput && (p < 4 || p >= 4 + 4 * 32)) || PadLocations.Any(g => g < 0 || g >= 32) || + if (PrimitiveVertices is < 1 or > 3 || Planes.Any(p => p != PointSizePlane && (p < 0 || p >= 4 + 4 * 32)) || + PrimitivePlanes.Any(p => p != NoOutput && (p < 4 || p >= 4 + 4 * 32)) || PadLocations.Any(g => g < 0 || g >= 32) || Planes.Distinct().Count() != Planes.Length || PrimitivePlanes.Where(p => p != NoOutput).Distinct().Count() != PrimitivePlanes.Count(p => p != NoOutput)) { throw new NotSupportedException($"mesh layout with a plane code out of range or repeated ({string.Join(", ", layout)})"); @@ -87,13 +102,13 @@ public MeshShaderInfo(bool isTask, int[] layout) throw new NotSupportedException($"mesh layout with a generic location both per vertex and per primitive ({string.Join(", ", layout)})"); } } - else if (Planes.Length != 0 || PrimitivePlanes.Length != 0 || PadLocations.Length != 0) + else if (hasTopology || Planes.Length != 0 || PrimitivePlanes.Length != 0 || PadLocations.Length != 0) { throw new NotSupportedException("task program with mesh output planes in its layout"); } PayloadWords = PowerOfTwo(Math.Max(4, (PayloadBytes + 3) / 4)); - PrimitiveWords = PowerOfTwo(4 + 3 * Math.Max(1, MaxPrimitives)); + PrimitiveWords = PowerOfTwo(4 + PrimitiveVertices * Chunks(Math.Max(1, MaxPrimitives)) * 32); } public static int Chunks(int n) diff --git a/dm/third_party/ryujinx/src/Spv.Generator/spirv.cs b/dm/third_party/ryujinx/src/Spv.Generator/spirv.cs index ccbe4714..cc80752f 100644 --- a/dm/third_party/ryujinx/src/Spv.Generator/spirv.cs +++ b/dm/third_party/ryujinx/src/Spv.Generator/spirv.cs @@ -161,6 +161,7 @@ public enum ExecutionMode DerivativeGroupLinearNV = 5290, OutputTrianglesNV = 5298, OutputPrimitivesEXT = 5270, + OutputLinesEXT = OutputLinesNV, OutputTrianglesEXT = 5298, PixelInterlockOrderedEXT = 5366, PixelInterlockUnorderedEXT = 5367, @@ -586,6 +587,8 @@ public enum BuiltIn TaskCountNV = 5274, PrimitiveCountNV = 5275, PrimitiveIndicesNV = 5276, + PrimitivePointIndicesEXT = 5294, + PrimitiveLineIndicesEXT = 5295, PrimitiveTriangleIndicesEXT = 5296, ClipDistancePerViewNV = 5277, CullDistancePerViewNV = 5278, From 66d7038f9dab8cab757240ca9f2ee220a179e635 Mon Sep 17 00:00:00 2001 From: seedds Date: Wed, 11 Nov 2026 20:50:51 +0100 Subject: [PATCH 4/6] shader: bound raw global memory lookup cost --- dm/gfx/tests/test_dmsh_raw_reuse.py | 119 +++++ .../CodeGen/Glsl/Declarations.cs | 2 +- .../CodeGen/Spirv/Declarations.cs | 5 +- .../StructuredIr/MemoryDefinition.cs | 6 +- .../Optimizations/GlobalToStorage.cs | 479 +++++++++++++++++- .../Translation/ResourceManager.cs | 6 + 6 files changed, 587 insertions(+), 30 deletions(-) create mode 100644 dm/gfx/tests/test_dmsh_raw_reuse.py diff --git a/dm/gfx/tests/test_dmsh_raw_reuse.py b/dm/gfx/tests/test_dmsh_raw_reuse.py new file mode 100644 index 00000000..a5c8ac5a --- /dev/null +++ b/dm/gfx/tests/test_dmsh_raw_reuse.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Raw-global lookups use the current hint ABI and reuse resolved entries. + +The first access tries the tagged 2 MiB-chunk hint before a looped table search. +Consecutive loads from the same base in one block use a small helper that +applies the cached entry's delta, while retaining a checked fallback when an +offset crosses a pool range. +""" +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from test_dmsh_ampere_mem import EXIT, Translated, inst # noqa: E402 + +TOOLS = Path(__file__).resolve().parents[1] / 'tools' +STORE = (0x000fe8000000c800, 0x0000080609008388) +LDG = (0x000ea200081e890f, 0x0000000405067981) +LDG128 = (LDG[0] | (3 << 9), (LDG[1] & ~0xff0000) | (8 << 16)) + + +def waiting(instruction): + """Make the instruction wait for every scoreboard dependency.""" + return (instruction[0] | 0x3f << 52, instruction[1]) + + +def s2r_tid(register): + """Read SR_TID.X into *register* using the fixture's known SM86 form.""" + return (0x000e260000002100, + 0x0000000000007919 | register << 16) + + +@unittest.skipUnless(os.environ.get('DMSH_TEST_XLATE'), + 'set DMSH_TEST_XLATE to the offline translator') +@unittest.skipUnless(shutil.which('spirv-dis'), 'needs spirv-dis') +class RawReuse(Translated): + def spv_dis(self, code): + sys.path.insert(0, str(TOOLS)) + import dmsh_geometry + + with tempfile.TemporaryDirectory(prefix='raw_reuse_') as directory: + work = Path(directory) + (work / 'k.bin').write_bytes( + b''.join(inst(*waiting(op)) for op in code) + inst(*EXIT)) + job = [dict(key='k', stage='cs', arch='sm86', + code=str(work / 'k.bin'), tex={}, img={}, attr={}, + fout={}, local=[32, 1, 1], debug=True, + nvn2_cs=[0, 0, 3, 0x40, 0], shared=0x1000)] + (work / 'jobs.json').write_text(json.dumps(job)) + run = subprocess.run( + [os.environ['DMSH_TEST_XLATE'], str(work / 'jobs.json'), + directory], capture_output=True, text=True) + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + dmsh_geometry.flatten_sets(work / 'k.spv') + if shutil.which('spirv-val'): + valid = subprocess.run( + ['spirv-val', '--target-env', 'vulkan1.3', + str(work / 'k.spv')], capture_output=True, text=True) + self.assertEqual(valid.returncode, 0, + valid.stdout + valid.stderr) + return subprocess.run( + ['spirv-dis', str(work / 'k.spv')], capture_output=True, + text=True, check=True).stdout + + @staticmethod + def functions(disassembly): + return {match.group(1): match.group(0) for match in re.finditer( + r'(%\w+) = OpFunction %int None .*?OpFunctionEnd', + disassembly, re.S)} + + @staticmethod + def main_function(disassembly): + return re.search( + r'%\w+ = OpFunction %void None .*?OpFunctionEnd', + disassembly, re.S).group(0) + + def test_loads_share_one_zero_initialized_last_entry(self): + second = (LDG[0], LDG[1] | 4 << 40) + disassembly = self.spv_dis( + (s2r_tid(9), s2r_tid(5), LDG, STORE, second, STORE)) + self.assertEqual(len(re.findall( + r'OpVariable \S+ Private %\w+', disassembly)), 1) + main = self.main_function(disassembly) + calls = re.findall(r'OpFunctionCall %int (%\w+)', main) + self.assertEqual(len(calls), 2) + self.assertEqual(calls[0], calls[1]) + full = self.functions(disassembly)[calls[0]] + self.assertIn('OpLoopMerge', full) + self.assertIn('OpConstant %int 8191', disassembly) + self.assertRegex(full, r'OpShiftRightLogical %int \S+ %int_21') + self.assertRegex(full, r'OpBitwiseAnd %int \S+ %int_8191') + self.assertRegex(full, r'OpShiftRightLogical %int \S+ %int_13') + + def test_same_base_words_use_the_small_helper(self): + stores = tuple((STORE[0], (STORE[1] & ~0xff000000) | + (register << 24) | (offset << 40)) + for register, offset in + ((8, 0), (9, 4), (10, 16), (11, 20))) + disassembly = self.spv_dis( + (s2r_tid(9), s2r_tid(5), LDG128, *stores)) + main = self.main_function(disassembly) + calls = re.findall(r'OpFunctionCall %int (%\w+)', main) + self.assertEqual(len(calls), 4) + self.assertNotEqual(calls[0], calls[1]) + self.assertEqual(calls[1:], [calls[1]] * 3) + functions = self.functions(disassembly) + self.assertIn('OpLoopMerge', functions[calls[0]]) + self.assertNotIn('OpLoopMerge', functions[calls[1]]) + self.assertIn('OpFunctionCall %int ' + calls[0], functions[calls[1]]) + + +if __name__ == '__main__': + unittest.main() diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs index a7b82e74..6230f6c0 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs @@ -331,7 +331,7 @@ private static void DeclareMemories(CodeGenContext context, IEnumerable _entries; private readonly Dictionary> _sharedEntries; + private readonly Dictionary<(Instruction, StorageKind), int> _rawLastBindings; + private readonly Dictionary<(Instruction, StorageKind), int> _rawSameBaseFunctions; private readonly HelperFunctionManager _hfm; + private BasicBlock _rawBlock; + private Instruction _rawInst; + private StorageKind _rawStorageKind; + private Operand _rawBaseLo; + private Operand _rawBaseHi; + private int _rawFirstOffset; public GtsContext(HelperFunctionManager hfm) { _entries = []; _sharedEntries = new Dictionary>(); + _rawLastBindings = new Dictionary<(Instruction, StorageKind), int>(); + _rawSameBaseFunctions = new Dictionary<(Instruction, StorageKind), int>(); _hfm = hfm; } + public int GetOrAddRawLastBinding(ResourceManager resourceManager, Operation operation) + { + var key = (operation.Inst, operation.StorageKind); + + if (!_rawLastBindings.TryGetValue(key, out int binding)) + { + binding = resourceManager.AddRawLastEntryMemory(); + _rawLastBindings.Add(key, binding); + } + + return binding; + } + + public bool TryGetRawSameBaseFunctionId(Operation operation, out int functionId) + { + return _rawSameBaseFunctions.TryGetValue((operation.Inst, operation.StorageKind), out functionId); + } + + public int AddRawSameBaseFunction(Operation operation, Function function) + { + int functionId = _hfm.AddFunction(function); + _rawSameBaseFunctions.Add((operation.Inst, operation.StorageKind), functionId); + return functionId; + } + + public bool MatchesRawBase(BasicBlock block, Operation operation, Operand baseLo, Operand baseHi, int constOffset) + { + if ((operation.Inst != Instruction.Load && + (operation.Inst != Instruction.Store || IsSmallInt(operation.StorageKind))) || + block != _rawBlock || + operation.Inst != _rawInst || + operation.StorageKind != _rawStorageKind || + !SameAddressOperand(block, baseLo, _rawBaseLo) || + !SameAddressOperand(block, baseHi, _rawBaseHi)) + { + return false; + } + + long delta = (long)constOffset - _rawFirstOffset; + return delta >= -4096 && delta < 4096; + } + + public void SetRawBase(BasicBlock block, Operation operation, Operand baseLo, Operand baseHi, int constOffset) + { + _rawBlock = block; + _rawInst = operation.Inst; + _rawStorageKind = operation.StorageKind; + _rawBaseLo = baseLo; + _rawBaseHi = baseHi; + _rawFirstOffset = constOffset; + } + public int AddFunction(Operation baseOp, bool isMultiTarget, IReadOnlyList targetCbs, Function function) { int functionId = _hfm.AddFunction(function); @@ -656,6 +718,32 @@ private static bool TryGenerateMultiTargetStorageOp( if (targetCbs.Count == 0) { gpuAccessor.Log($"Global memory operation \"{operation.Inst}\" traced to no storage buffer: it goes through the memory pool table."); + + DecomposeRawAddress(block, operation.GetSource(0), operation.GetSource(1), out Operand baseLo, out Operand baseHi, out int constOffset); + + if (gtsContext.MatchesRawBase(block, operation, baseLo, baseHi, constOffset)) + { + if (gtsContext.TryGetRawSameBaseFunctionId(operation, out functionId)) + { + return true; + } + + if (!gtsContext.TryGetFunctionId(operation, isMultiTarget: true, [], out int fallbackFunctionId)) + { + return false; + } + + int lastBinding = gtsContext.GetOrAddRawLastBinding(resourceManager, operation); + return TryGenerateRawSameBaseStorageOp( + gtsContext, + resourceManager, + operation, + lastBinding, + fallbackFunctionId, + out functionId); + } + + gtsContext.SetRawBase(block, operation, baseLo, baseHi, constOffset); } if (gtsContext.TryGetFunctionId(operation, isMultiTarget: true, targetCbs, out functionId)) @@ -783,8 +871,9 @@ private static bool TryGenerateMultiTargetStorageOp( // slot, which no constant buffer of the game has (dmsh_build.py turns it into DMSH_KIND_GMEM_MAP). 1 << GmemMapLog2 entries of GmemMapEntryWords words each, // {start lo, start hi, end lo, end hi, delta lo, delta hi, 0, 0}, sorted by start; entry 0 is empty (start 0, end 0) and unused entries start at ~0 public const int GmemMapCbSlot = 0xff; - private const int GmemMapLog2 = 12; + private const int GmemMapLog2 = 13; private const int GmemMapEntryWords = 8; + private const int GmemHintShift = 21; // nxnative: the write log (dmsh_abi.h DMSH_GMEM_LOG_*), reserved under the same sentinel slot at offset 1: the set of guest pages (address >> 12) the stores and atomics wrote, // as an open-addressing hash set of 1 << GmemLogLog2 words from word 1 (0 = empty), word 0 set when a page found no free slot in GmemLogProbes tries. The host cannot know where @@ -853,33 +942,76 @@ Operand MapWord(Operand entry, int word) return context.Load(StorageKind.StorageBuffer, mapBinding, Const(0), context.IAdd(entry, Const(word))); } - // The last entry whose start is at or below the address (entry 0 starts at 0, so there always is one) - Operand index = Const(0); - - for (int step = 1 << (GmemMapLog2 - 1); step != 0; step >>= 1) - { - Operand candidate = context.IAdd(index, Const(step)); - Operand entry = context.IMultiply(candidate, Const(GmemMapEntryWords)); - Operand startLow = MapWord(entry, 0); - Operand startHigh = MapWord(entry, 1); - - Operand atOrBelow = context.BitwiseOr( - context.ICompareLessUnsigned(startHigh, addrHigh), - context.BitwiseAnd(context.ICompareEqual(startHigh, addrHigh), context.ICompareLessOrEqualUnsigned(startLow, addrLow))); - - index = context.ConditionalSelect(atOrBelow, candidate, index); - } - - Operand found = context.IMultiply(index, Const(GmemMapEntryWords)); - Operand endLow = MapWord(found, 2); - Operand endHigh = MapWord(found, 3); - - Operand inPool = context.BitwiseOr( - context.ICompareLessUnsigned(addrHigh, endHigh), - context.BitwiseAnd(context.ICompareEqual(addrHigh, endHigh), context.ICompareLessUnsigned(addrLow, endLow))); + Operand AtOrBelow(Operand high, Operand low) + { + return context.BitwiseOr( + context.ICompareLessUnsigned(high, addrHigh), + context.BitwiseAnd(context.ICompareEqual(high, addrHigh), context.ICompareLessOrEqualUnsigned(low, addrLow))); + } + + Operand Below(Operand high, Operand low) + { + return context.BitwiseOr( + context.ICompareLessUnsigned(addrHigh, high), + context.BitwiseAnd(context.ICompareEqual(addrHigh, high), context.ICompareLessUnsigned(addrLow, low))); + } + + Operand selected = Local(); + Operand lblFound = Label(); + Operand lblChosen = Label(); + + // Try the entry this helper chose last in this invocation before + // searching the table. Entry zero is empty, so the first lookup + // still searches. The host binds one disjoint table for the draw. + int lastBinding = gtsContext.GetOrAddRawLastBinding(resourceManager, operation); + Operand last = context.Load(StorageKind.LocalMemory, lastBinding); + Operand lastEntry = context.IMultiply(last, Const(GmemMapEntryWords)); + Operand lastHolds = context.BitwiseAnd( + AtOrBelow(MapWord(lastEntry, 1), MapWord(lastEntry, 0)), + Below(MapWord(lastEntry, 3), MapWord(lastEntry, 2))); + context.Copy(selected, last); + context.BranchIfTrue(lblChosen, lastHolds); + + // A tagged 2 MiB-chunk hint names an entry which holds the whole + // chunk. Collisions and ranges which cover only part of a chunk + // take the search below. + Operand chunk = context.BitwiseOr( + context.ShiftLeft(addrHigh, Const(32 - GmemHintShift)), + context.ShiftRightU32(addrLow, Const(GmemHintShift))); + Operand indexMask = Const((1 << GmemMapLog2) - 1); + Operand hint = MapWord( + context.IMultiply(context.BitwiseAnd(chunk, indexMask), Const(GmemMapEntryWords)), 6); + Operand hintIndex = context.BitwiseAnd(hint, indexMask); + Operand hintFits = context.BitwiseAnd( + context.BitwiseAnd( + context.ICompareNotEqual(hintIndex, Const(0)), + context.ICompareEqual(context.ShiftRightU32(hint, Const(GmemMapLog2)), chunk)), + context.ICompareLessUnsigned(addrHigh, Const(1 << (GmemHintShift - GmemMapLog2)))); + context.Copy(selected, hintIndex); + context.BranchIfTrue(lblFound, hintFits); + // The last entry whose start is at or below the address (entry 0 starts at 0, so there always is one) + Operand index = Local(); + Operand step = Local(); + Operand lblSearch = Label(); + context.Copy(index, Const(0)); + context.Copy(step, Const(1 << (GmemMapLog2 - 1))); + context.MarkLabel(lblSearch); + Operand candidate = context.IAdd(index, step); + Operand entry = context.IMultiply(candidate, Const(GmemMapEntryWords)); + context.Copy(index, context.ConditionalSelect(AtOrBelow(MapWord(entry, 1), MapWord(entry, 0)), candidate, index)); + context.Copy(step, context.ShiftRightU32(step, Const(1))); + context.BranchIfTrue(lblSearch, context.ICompareNotEqual(step, Const(0))); + + context.Copy(selected, index); + context.MarkLabel(lblFound); + context.Store(StorageKind.LocalMemory, lastBinding, selected); + Operand found = context.IMultiply(selected, Const(GmemMapEntryWords)); Operand lblSkip = Label(); - context.BranchIfFalse(lblSkip, inPool); + context.BranchIfFalse(lblSkip, Below(MapWord(found, 3), MapWord(found, 2))); + + context.MarkLabel(lblChosen); + found = context.IMultiply(selected, Const(GmemMapEntryWords)); if (writes) { @@ -967,6 +1099,7 @@ Operand MapWord(Operand entry, int word) } context.MarkLabel(lblSkip); + context.Store(StorageKind.LocalMemory, lastBinding, Const(0)); bool returnsValue = operation.Dest != null; @@ -991,6 +1124,300 @@ Operand MapWord(Operand entry, int word) return true; } + private static bool TryGenerateRawSameBaseStorageOp( + GtsContext gtsContext, + ResourceManager resourceManager, + Operation operation, + int lastBinding, + int fallbackFunctionId, + out int functionId) + { + functionId = 0; + + if (!resourceManager.TryGetStorageBufferBinding(GmemMapCbSlot, 0, write: false, out int mapBinding)) + { + return false; + } + + bool isSmall = IsSmallInt(operation.StorageKind); + int inArgumentsCount = operation.Inst == Instruction.Store ? 3 : 2; + + EmitterContext context = new(); + + Operand addrLow = Argument(0); + Operand addrHigh = Argument(1); + + Operand MapWord(Operand entry, int word) + { + return context.Load(StorageKind.StorageBuffer, mapBinding, Const(0), context.IAdd(entry, Const(word))); + } + + Operand last = context.Load(StorageKind.LocalMemory, lastBinding); + Operand found = context.IMultiply(last, Const(GmemMapEntryWords)); + Operand startsAtOrBelow = context.BitwiseOr( + context.ICompareLessUnsigned(MapWord(found, 1), addrHigh), + context.BitwiseAnd( + context.ICompareEqual(MapWord(found, 1), addrHigh), + context.ICompareLessOrEqualUnsigned(MapWord(found, 0), addrLow))); + Operand endsAbove = context.BitwiseOr( + context.ICompareLessUnsigned(addrHigh, MapWord(found, 3)), + context.BitwiseAnd( + context.ICompareEqual(addrHigh, MapWord(found, 3)), + context.ICompareLessUnsigned(addrLow, MapWord(found, 2)))); + Operand lblFallback = Label(); + context.BranchIfFalse(lblFallback, context.BitwiseAnd(startsAtOrBelow, endsAbove)); + + Operand wordLow = isSmall ? context.BitwiseAnd(addrLow, Const(~3)) : addrLow; + Operand hostLow = context.IAdd(wordLow, MapWord(found, 4)); + Operand carry = context.ConditionalSelect(context.ICompareLessUnsigned(hostLow, wordLow), Const(1), Const(0)); + Operand hostHigh = context.IAdd(context.IAdd(addrHigh, MapWord(found, 5)), carry); + + Operand resultValue = null; + + if (operation.Inst == Instruction.Load) + { + resultValue = context.Load(StorageKind.GlobalMemory, hostLow, hostHigh); + + if (isSmall) + { + Operand bitOffset = HelperFunctionManager.GetBitOffset(context, addrLow); + + resultValue = operation.StorageKind switch + { + StorageKind.GlobalMemoryS8 => context.BitfieldExtractS32(context.ShiftRightS32(resultValue, bitOffset), Const(0), Const(8)), + StorageKind.GlobalMemoryS16 => context.BitfieldExtractS32(context.ShiftRightS32(resultValue, bitOffset), Const(0), Const(16)), + StorageKind.GlobalMemoryU8 => context.BitwiseAnd(context.ShiftRightU32(resultValue, bitOffset), Const(byte.MaxValue)), + _ => context.BitwiseAnd(context.ShiftRightU32(resultValue, bitOffset), Const(ushort.MaxValue)), + }; + } + } + else + { + context.Store(StorageKind.GlobalMemory, hostLow, hostHigh, Argument(2)); + } + + if (resultValue != null) + { + context.Return(resultValue); + } + else + { + context.Return(); + } + + context.MarkLabel(lblFallback); + + bool returnsValue = operation.Dest != null; + + if (returnsValue) + { + context.Return(context.Call(fallbackFunctionId, true, addrLow, addrHigh)); + } + else + { + context.Call(fallbackFunctionId, false, addrLow, addrHigh, Argument(2)); + context.Return(); + } + + Function function = new( + ControlFlowGraph.Create(context.GetOperations()).Blocks, + GetFunctionName(operation, isMultiTarget: true, []) + "RawSameBase", + returnsValue, + inArgumentsCount, + 0); + + functionId = gtsContext.AddRawSameBaseFunction(operation, function); + + return true; + } + + private static void DecomposeRawAddress( + BasicBlock block, + Operand addrLow, + Operand addrHigh, + out Operand baseLo, + out Operand baseHi, + out int constOffset) + { + addrLow = Utils.FindLastOperation(addrLow, block); + addrHigh = Utils.FindLastOperation(addrHigh, block); + + if (addrLow.AsgOp is Operation addLo && + addLo.Inst == Instruction.Add && + addrHigh.AsgOp is Operation addHi && + addHi.Inst == Instruction.Add) + { + Operand lo0 = addLo.GetSource(0); + Operand lo1 = addLo.GetSource(1); + Operand bLo = null; + Operand cOp = null; + + if (lo0.Type == OperandType.LocalVariable && lo1.Type == OperandType.Constant) + { + bLo = lo0; + cOp = lo1; + } + else if (lo0.Type == OperandType.Constant && lo1.Type == OperandType.LocalVariable) + { + bLo = lo1; + cOp = lo0; + } + + if (bLo != null) + { + Operand hi0 = Utils.FindLastOperation(addHi.GetSource(0), block); + Operand hi1 = Utils.FindLastOperation(addHi.GetSource(1), block); + Operation hi0Op = hi0.AsgOp as Operation; + Operation hi1Op = hi1.AsgOp as Operation; + + if (hi1Op?.Inst == Instruction.ConditionalSelect && hi0Op?.Inst != Instruction.ConditionalSelect) + { + baseLo = Utils.FindLastOperation(bLo, block); + baseHi = hi0; + constOffset = UnwrapScaledIndex(block, ref baseLo, ref baseHi) ? 0 : cOp.Value; + return; + } + else if (hi0Op?.Inst == Instruction.ConditionalSelect && hi1Op?.Inst != Instruction.ConditionalSelect) + { + baseLo = Utils.FindLastOperation(bLo, block); + baseHi = hi1; + constOffset = UnwrapScaledIndex(block, ref baseLo, ref baseHi) ? 0 : cOp.Value; + return; + } + } + } + + baseLo = addrLow; + baseHi = addrHigh; + constOffset = 0; + UnwrapScaledIndex(block, ref baseLo, ref baseHi); + } + + private static bool UnwrapScaledIndex(BasicBlock block, ref Operand baseLo, ref Operand baseHi) + { + if (baseLo.AsgOp is not Operation addLo || addLo.Inst != Instruction.Add || + baseHi.AsgOp is not Operation addHi || addHi.Inst != Instruction.Add) + { + return false; + } + + Operand hi0 = Utils.FindLastOperation(addHi.GetSource(0), block); + Operand hi1 = Utils.FindLastOperation(addHi.GetSource(1), block); + Operation hi0Op = hi0.AsgOp as Operation; + Operation hi1Op = hi1.AsgOp as Operation; + Operation hiSumOp = null; + + if (hi1Op?.Inst == Instruction.ConditionalSelect && hi0Op?.Inst == Instruction.Add) + { + hiSumOp = hi0Op; + } + else if (hi0Op?.Inst == Instruction.ConditionalSelect && hi1Op?.Inst == Instruction.Add) + { + hiSumOp = hi1Op; + } + + if (hiSumOp == null) + { + return false; + } + + Operand lo0 = Utils.FindLastOperation(addLo.GetSource(0), block); + Operand lo1 = Utils.FindLastOperation(addLo.GetSource(1), block); + Operand hs0 = Utils.FindLastOperation(hiSumOp.GetSource(0), block); + Operand hs1 = Utils.FindLastOperation(hiSumOp.GetSource(1), block); + + static bool IsMulLo(Operand op) => (op.AsgOp as Operation)?.Inst is Instruction.Multiply or Instruction.ShiftLeft; + static bool IsMulHi(Operand op) => (op.AsgOp as Operation)?.Inst is Instruction.MultiplyHighU32 or Instruction.MultiplyHighS32 or Instruction.ShiftRightU32; + + Operand arrLo = null; + Operand arrHi = null; + + if (IsMulLo(lo0) && !IsMulLo(lo1)) + { + arrLo = lo1; + } + else if (IsMulLo(lo1) && !IsMulLo(lo0)) + { + arrLo = lo0; + } + + if (IsMulHi(hs0) && !IsMulHi(hs1)) + { + arrHi = hs1; + } + else if (IsMulHi(hs1) && !IsMulHi(hs0)) + { + arrHi = hs0; + } + + if (arrLo != null && arrHi != null) + { + baseLo = arrLo; + baseHi = arrHi; + return true; + } + + return false; + } + + private static bool SameAddressOperand(BasicBlock block, Operand a, Operand b, int depth = 0) + { + if (a == null || b == null) + { + return false; + } + + a = Utils.FindLastOperation(a, block); + b = Utils.FindLastOperation(b, block); + + if (a == b) + { + return true; + } + + if (a.Type == OperandType.Constant && b.Type == OperandType.Constant) + { + return a.Value == b.Value; + } + + if (depth >= 4 || a.AsgOp is not Operation opA || b.AsgOp is not Operation opB) + { + return false; + } + + if (opA.Inst != opB.Inst || opA.StorageKind != opB.StorageKind || opA.SourcesCount != opB.SourcesCount) + { + return false; + } + + switch (opA.Inst) + { + case Instruction.Add: + case Instruction.Multiply: + case Instruction.MultiplyHighS32: + case Instruction.MultiplyHighU32: + case Instruction.ShiftLeft: + case Instruction.ShiftRightU32: + case Instruction.ConditionalSelect: + case Instruction.CompareEqual: + case Instruction.CompareLessU32: + case Instruction.CompareGreaterU32: + break; + default: + return false; + } + + for (int i = 0; i < opA.SourcesCount; i++) + { + if (!SameAddressOperand(block, opA.GetSource(i), opB.GetSource(i), depth + 1)) + { + return false; + } + } + + return true; + } + private static uint PackCbSlotAndOffset(int cbSlot, int cbOffset) { return (uint)((ushort)cbSlot | ((ushort)cbOffset << 16)); diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs index 0a4c0ff4..2fbad681 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs @@ -159,6 +159,12 @@ public void SetVertexAsComputeLocalMemories(ShaderStage stage, InputTopology inp } } + // nxnative: the pool table entry a raw global memory helper chose last in this invocation (GlobalToStorage.TryGenerateRawStorageOp), one word for each helper, zero at the start + public int AddRawLastEntryMemory() + { + return Properties.AddLocalMemory(new MemoryDefinition($"gmem_last_{Properties.LocalMemories.Count}", AggregateType.U32, 1, zeroInit: true)); + } + private int AddMemoryDefinition(string name, AggregateType type, int arrayLength = 1) { return Properties.AddLocalMemory(new MemoryDefinition(name, type, arrayLength)); From ab421bcefc12616d75c22f07ad89dc0b8d43e37b Mon Sep 17 00:00:00 2001 From: seedds Date: Wed, 11 Nov 2026 21:13:00 +0100 Subject: [PATCH 5/6] shader: vectorize wide raw global loads --- dm/gfx/tests/test_dmsh_raw_reuse.py | 49 +- .../CodeGen/Spirv/Instructions.cs | 47 ++ .../Instructions/InstEmitAmpereMem.cs | 15 +- .../Instructions/InstEmitMemory.cs | 10 + .../IntermediateRepresentation/Operation.cs | 6 + .../IntermediateRepresentation/StorageKind.cs | 1 + .../Translation/EmitterContext.cs | 4 + .../Optimizations/GlobalToStorage.cs | 517 +++++------------- .../Translation/ResourceManager.cs | 15 + 9 files changed, 279 insertions(+), 385 deletions(-) diff --git a/dm/gfx/tests/test_dmsh_raw_reuse.py b/dm/gfx/tests/test_dmsh_raw_reuse.py index a5c8ac5a..30b644dd 100644 --- a/dm/gfx/tests/test_dmsh_raw_reuse.py +++ b/dm/gfx/tests/test_dmsh_raw_reuse.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -"""Raw-global lookups use the current hint ABI and reuse resolved entries. +"""Raw-global lookups use the current hint ABI and vectorize wide loads. The first access tries the tagged 2 MiB-chunk hint before a looped table search. -Consecutive loads from the same base in one block use a small helper that -applies the cached entry's delta, while retaining a checked fallback when an -offset crosses a pool range. +Later accesses first check the entry cached for this invocation. LDG.64 and +LDG.128 resolve once and perform one aligned vector load, including zeroing all +follower words when the address misses every pool. """ import json import os @@ -22,7 +22,8 @@ TOOLS = Path(__file__).resolve().parents[1] / 'tools' STORE = (0x000fe8000000c800, 0x0000080609008388) LDG = (0x000ea200081e890f, 0x0000000405067981) -LDG128 = (LDG[0] | (3 << 9), (LDG[1] & ~0xff0000) | (8 << 16)) +VEC128 = ((LDG[0] & ~(7 << 9)) | (6 << 9), + (LDG[1] & ~0xff0000) | (12 << 16)) def waiting(instruction): @@ -97,22 +98,42 @@ def test_loads_share_one_zero_initialized_last_entry(self): self.assertRegex(full, r'OpBitwiseAnd %int \S+ %int_8191') self.assertRegex(full, r'OpShiftRightLogical %int \S+ %int_13') - def test_same_base_words_use_the_small_helper(self): + def test_128_bit_load_uses_one_vector_helper(self): stores = tuple((STORE[0], (STORE[1] & ~0xff000000) | (register << 24) | (offset << 40)) for register, offset in - ((8, 0), (9, 4), (10, 16), (11, 20))) + ((12, 0), (13, 4), (14, 16), (15, 20))) disassembly = self.spv_dis( - (s2r_tid(9), s2r_tid(5), LDG128, *stores)) + (s2r_tid(9), s2r_tid(5), VEC128, *stores)) main = self.main_function(disassembly) calls = re.findall(r'OpFunctionCall %int (%\w+)', main) - self.assertEqual(len(calls), 4) - self.assertNotEqual(calls[0], calls[1]) - self.assertEqual(calls[1:], [calls[1]] * 3) + self.assertEqual(len(calls), 1) functions = self.functions(disassembly) - self.assertIn('OpLoopMerge', functions[calls[0]]) - self.assertNotIn('OpLoopMerge', functions[calls[1]]) - self.assertIn('OpFunctionCall %int ' + calls[0], functions[calls[1]]) + helper = functions[calls[0]] + self.assertIn('OpLoopMerge', helper) + self.assertRegex(helper, r'OpLoad %v4uint \S+ Aligned 4') + self.assertGreaterEqual( + len(re.findall(r'OpStore \S+ %uint_0', helper)), 4, + 'a miss must zero all three follower words and the last-entry cache') + + def test_adjacent_vectors_do_not_share_follower_words(self): + first = (VEC128[0] & ~0xf, VEC128[1]) + second = (first[0], + (first[1] & ~(0xff << 16) & ~(0xffffff << 40)) | + (16 << 16) | (0x40 << 40)) + keep_first = (STORE[0], (STORE[1] & ~0xff000000) | (12 << 24)) + keep_second = (STORE[0], + (STORE[1] & ~0xff000000 & ~(0xffffff << 40)) | + (18 << 24) | (4 << 40)) + disassembly = self.spv_dis( + (s2r_tid(9), s2r_tid(5), first, second, + keep_first, keep_second)) + calls = re.findall( + r'OpFunctionCall %int (%\w+)', self.main_function(disassembly)) + self.assertEqual(len(calls), 2) + functions = self.functions(disassembly) + self.assertEqual(sum('OpLoad %v4uint' in functions[call] + for call in set(calls)), 1) if __name__ == '__main__': diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs index 48df704e..c52321d6 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs @@ -1800,6 +1800,11 @@ private static OperationResult GenerateAtomicMemoryCas(CodeGenContext context, A private static OperationResult GenerateLoadOrStore(CodeGenContext context, AstOperation operation, bool isStore) { + if (!isStore && operation.StorageKind == StorageKind.GlobalMemoryVector) + { + return GenerateRawVectorLoad(context, operation); + } + // nxnative (after azmasm/switch2mac, MIT): a Switch 2 mesh program's store into its per-vertex / per-primitive planes goes straight into the mesh outputs // (CodeGenContext.MeshPlaneStore, written by SpirvGenerator.GenerateMeshEntry), so the values are not held twice if (isStore && operation.StorageKind == StorageKind.SharedMemory && context.Properties.Mesh is { IsMesh: true } mesh && operation.SourcesCount == 3 && @@ -1840,6 +1845,48 @@ private static OperationResult GenerateLoadOrStore(CodeGenContext context, AstOp } } + // Load an aligned 64- or 128-bit raw value through one physical + // pointer. Word zero is returned; the rest go to the private array + // consumed by the original follower loads. + private static OperationResult GenerateRawVectorLoad(CodeGenContext context, AstOperation operation) + { + if (operation.GetSource(2) is not AstOperand { Type: OperandType.Constant } countOperand || + operation.GetSource(3) is not AstOperand { Type: OperandType.Constant } memoryOperand) + { + throw new InvalidOperationException( + $"The word count and local memory of a {operation.StorageKind} load must be constants."); + } + + int count = countOperand.Value; + SpvInstruction vectorType = context.TypeVector(context.TypeU32(), count); + SpvInstruction addressWords = context.CompositeConstruct( + context.TypeVector(context.TypeU32(), 2), + context.Get(AggregateType.U32, operation.GetSource(0)), + context.Get(AggregateType.U32, operation.GetSource(1))); + SpvInstruction pointer = context.ConvertUToPtr( + context.TypePointer(StorageClass.PhysicalStorageBuffer, vectorType), + context.Bitcast(context.TypeInt(64, 0), addressWords)); + + SpvInstruction value = context.Load(vectorType, pointer, MemoryAccessMask.Aligned); + value.AddOperand((SpvLiteralInteger)4); + SpvInstruction wordPointerType = context.TypePointer(StorageClass.Private, context.TypeU32()); + + for (int word = 1; word < count; word++) + { + SpvInstruction element = context.AccessChain( + wordPointerType, + context.LocalMemories[memoryOperand.Value], + context.Constant(context.TypeS32(), word - 1)); + context.Store( + element, + context.CompositeExtract(context.TypeU32(), value, (SpvLiteralInteger)word)); + } + + return new OperationResult( + AggregateType.U32, + context.CompositeExtract(context.TypeU32(), value, (SpvLiteralInteger)0)); + } + private static SpvInstruction GetStoragePointer(CodeGenContext context, AstOperation operation, out AggregateType varType) { StorageKind storageKind = operation.StorageKind; diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMem.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMem.cs index 69db51cc..8fd63fcc 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMem.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMem.cs @@ -256,6 +256,10 @@ public static void ALdg(EmitterContext context) (los[index], his[index]) = AGlobalAddress(context, x, 90, 32, index * 4); } + bool vector = count > 1 && storageKind == StorageKind.GlobalMemory && + x.Rd + count <= RegisterConsts.RegisterZeroIndex; + int group = vector ? ++context.GlobalVectorGroups : 0; + for (int index = 0; index < count; index++) { if (x.Rd + index >= RegisterConsts.RegisterZeroIndex) @@ -263,7 +267,16 @@ public static void ALdg(EmitterContext context) break; } - context.Copy(Register(x.Rd + index, RegisterType.Gpr), context.Load(storageKind, los[index], his[index])); + Operand value = context.Load(storageKind, los[index], his[index]); + + if (vector && value.AsgOp is Operation load) + { + load.GlobalVectorCount = count; + load.GlobalVectorWord = index; + load.GlobalVectorGroup = group; + } + + context.Copy(Register(x.Rd + index, RegisterType.Gpr), value); } if (lblSkip != null) diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs index 06a91796..d04245d3 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs @@ -371,6 +371,9 @@ private static void EmitLdg( (_, Operand addrHigh) = Get40BitsAddress(context, new Register(ra, RegisterType.Gpr), extended, offset); Operand srcA = context.Copy(new Operand(new Register(ra, RegisterType.Gpr))); + bool vector = count > 1 && storageKind == StorageKind.GlobalMemory && + rd + count <= RegisterConsts.RegisterZeroIndex; + int group = vector ? ++context.GlobalVectorGroups : 0; for (int index = 0; index < count; index++) { @@ -383,6 +386,13 @@ private static void EmitLdg( Operand value = context.Load(storageKind, context.IAdd(srcA, Const(offset + index * 4)), addrHigh); + if (vector && value.AsgOp is Operation load) + { + load.GlobalVectorCount = count; + load.GlobalVectorWord = index; + load.GlobalVectorGroup = group; + } + context.Copy(Register(dest), value); } } diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/Operation.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/Operation.cs index dc2f56e7..092d903e 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/Operation.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/Operation.cs @@ -10,6 +10,12 @@ class Operation : INode public bool ForcePrecise { get; set; } + // Words emitted from one LDG.64/LDG.128. The group distinguishes + // adjacent vector loads with the same width. + public int GlobalVectorCount { get; set; } + public int GlobalVectorWord { get; set; } + public int GlobalVectorGroup { get; set; } + private Operand[] _dests; public Operand Dest diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/StorageKind.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/StorageKind.cs index 2b39e85e..0e70f35a 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/StorageKind.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/StorageKind.cs @@ -18,6 +18,7 @@ enum StorageKind GlobalMemoryS16, // TODO: Remove this and store type as a field on the Operation class itself. GlobalMemoryU8, // TODO: Remove this and store type as a field on the Operation class itself. GlobalMemoryU16, // TODO: Remove this and store type as a field on the Operation class itself. + GlobalMemoryVector, // nxnative: a raw vector load at a host GPU address: low, high, word count, and local memory for words 1 and up. } static class StorageKindExtensions diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs index 4152df40..92564001 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs @@ -22,6 +22,10 @@ class EmitterContext public Block CurrBlock { get; set; } public InstOp CurrOp { get; set; } + // Zero means no vector access; each emitted LDG vector gets a distinct + // positive group so follower scans cannot consume another load's word. + public int GlobalVectorGroups { get; set; } + public int OperationsCount => _operations.Count; private readonly struct BrxTarget diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs index 5e527b92..470b4438 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs @@ -74,74 +74,41 @@ public bool Equals(LsKey other) private readonly List _entries; private readonly Dictionary> _sharedEntries; private readonly Dictionary<(Instruction, StorageKind), int> _rawLastBindings; - private readonly Dictionary<(Instruction, StorageKind), int> _rawSameBaseFunctions; + private readonly Dictionary _rawVectorFunctions; private readonly HelperFunctionManager _hfm; - private BasicBlock _rawBlock; - private Instruction _rawInst; - private StorageKind _rawStorageKind; - private Operand _rawBaseLo; - private Operand _rawBaseHi; - private int _rawFirstOffset; public GtsContext(HelperFunctionManager hfm) { _entries = []; _sharedEntries = new Dictionary>(); _rawLastBindings = new Dictionary<(Instruction, StorageKind), int>(); - _rawSameBaseFunctions = new Dictionary<(Instruction, StorageKind), int>(); + _rawVectorFunctions = []; _hfm = hfm; } - public int GetOrAddRawLastBinding(ResourceManager resourceManager, Operation operation) + public bool TryGetRawVectorFunctionId(int count, out int functionId) { - var key = (operation.Inst, operation.StorageKind); - - if (!_rawLastBindings.TryGetValue(key, out int binding)) - { - binding = resourceManager.AddRawLastEntryMemory(); - _rawLastBindings.Add(key, binding); - } - - return binding; - } - - public bool TryGetRawSameBaseFunctionId(Operation operation, out int functionId) - { - return _rawSameBaseFunctions.TryGetValue((operation.Inst, operation.StorageKind), out functionId); + return _rawVectorFunctions.TryGetValue(count, out functionId); } - public int AddRawSameBaseFunction(Operation operation, Function function) + public int AddRawVectorFunction(int count, Function function) { int functionId = _hfm.AddFunction(function); - _rawSameBaseFunctions.Add((operation.Inst, operation.StorageKind), functionId); + _rawVectorFunctions.Add(count, functionId); return functionId; } - public bool MatchesRawBase(BasicBlock block, Operation operation, Operand baseLo, Operand baseHi, int constOffset) + public int GetOrAddRawLastBinding(ResourceManager resourceManager, Operation operation) { - if ((operation.Inst != Instruction.Load && - (operation.Inst != Instruction.Store || IsSmallInt(operation.StorageKind))) || - block != _rawBlock || - operation.Inst != _rawInst || - operation.StorageKind != _rawStorageKind || - !SameAddressOperand(block, baseLo, _rawBaseLo) || - !SameAddressOperand(block, baseHi, _rawBaseHi)) + var key = (operation.Inst, operation.StorageKind); + + if (!_rawLastBindings.TryGetValue(key, out int binding)) { - return false; + binding = resourceManager.AddRawLastEntryMemory(); + _rawLastBindings.Add(key, binding); } - long delta = (long)constOffset - _rawFirstOffset; - return delta >= -4096 && delta < 4096; - } - - public void SetRawBase(BasicBlock block, Operation operation, Operand baseLo, Operand baseHi, int constOffset) - { - _rawBlock = block; - _rawInst = operation.Inst; - _rawStorageKind = operation.StorageKind; - _rawBaseLo = baseLo; - _rawBaseHi = baseHi; - _rawFirstOffset = constOffset; + return binding; } public int AddFunction(Operation baseOp, bool isMultiTarget, IReadOnlyList targetCbs, Function function) @@ -443,15 +410,116 @@ private static LinkedListNode ReplaceGlobalMemoryWithStorage( targetLanguage, block, operation, - out int functionId)) + out int functionId, + out bool raw)) { return null; } + if (raw && + operation.Inst == Instruction.Load && + operation.StorageKind == StorageKind.GlobalMemory && + operation.GlobalVectorCount > 1 && + operation.GlobalVectorWord == 0 && + TryGetRawVectorFunction( + gtsContext, + resourceManager, + gpuAccessor, + operation, + out int vectorFunctionId)) + { + return GenerateRawVectorLoad(resourceManager, node, operation, vectorFunctionId); + } + return GenerateCallStorageOp(node, operation, null, functionId); } } + private static LinkedListNode GenerateRawVectorLoad( + ResourceManager resourceManager, + LinkedListNode node, + Operation operation, + int functionId) + { + int count = operation.GlobalVectorCount; + int vectorMemoryId = resourceManager.GetRawVectorMemory(); + + node = GenerateCallStorageOp(node, operation, null, functionId); + LinkedListNode insertAfter = node; + + for (LinkedListNode next = node.Next; next != null; next = next.Next) + { + if (next.Value is not Operation follower) + { + continue; + } + + if (follower.Inst == Instruction.Load && + follower.StorageKind == StorageKind.GlobalMemory && + follower.GlobalVectorCount == count && + follower.GlobalVectorGroup == operation.GlobalVectorGroup && + follower.GlobalVectorWord > 0) + { + Operand word = Local(); + Operation load = new( + Instruction.Load, + StorageKind.LocalMemory, + word, + Const(vectorMemoryId), + Const(follower.GlobalVectorWord - 1)); + insertAfter = insertAfter.List.AddAfter(insertAfter, load); + + Operand dest = follower.Dest; + follower.TurnIntoCopy(word); + word.UseOps.Remove(follower); + follower.Dest = null; + next.Value = new Operation(Instruction.Copy, dest, word); + + if (follower.GlobalVectorWord == count - 1) + { + break; + } + + continue; + } + + if (follower.GlobalVectorCount != 0 || + follower.StorageKind != StorageKind.None || + follower.Inst == Instruction.Call || + follower.Inst == Instruction.Barrier || + follower.Inst == Instruction.MemoryBarrier || + follower.Inst.IsAtomic()) + { + break; + } + } + + return node; + } + + private static bool TryGetRawVectorFunction( + GtsContext gtsContext, + ResourceManager resourceManager, + IGpuAccessor gpuAccessor, + Operation operation, + out int functionId) + { + int count = operation.GlobalVectorCount; + + if (gtsContext.TryGetRawVectorFunctionId(count, out functionId)) + { + return true; + } + + return TryGenerateRawStorageOp( + gtsContext, + resourceManager, + gpuAccessor, + operation, + out functionId, + count); + } + private static bool CanUseInlineStorageOp(Operation operation, TargetLanguage targetLanguage) { if (operation.StorageKind != StorageKind.GlobalMemory) @@ -651,11 +719,13 @@ private static bool TryGenerateMultiTargetStorageOp( TargetLanguage targetLanguage, BasicBlock block, Operation operation, - out int functionId) + out int functionId, + out bool raw) { Queue phis = new(); HashSet visited = []; List targetCbs = []; + raw = false; Operand globalAddress = operation.GetSource(0); @@ -717,33 +787,8 @@ private static bool TryGenerateMultiTargetStorageOp( if (targetCbs.Count == 0) { + raw = true; gpuAccessor.Log($"Global memory operation \"{operation.Inst}\" traced to no storage buffer: it goes through the memory pool table."); - - DecomposeRawAddress(block, operation.GetSource(0), operation.GetSource(1), out Operand baseLo, out Operand baseHi, out int constOffset); - - if (gtsContext.MatchesRawBase(block, operation, baseLo, baseHi, constOffset)) - { - if (gtsContext.TryGetRawSameBaseFunctionId(operation, out functionId)) - { - return true; - } - - if (!gtsContext.TryGetFunctionId(operation, isMultiTarget: true, [], out int fallbackFunctionId)) - { - return false; - } - - int lastBinding = gtsContext.GetOrAddRawLastBinding(resourceManager, operation); - return TryGenerateRawSameBaseStorageOp( - gtsContext, - resourceManager, - operation, - lastBinding, - fallbackFunctionId, - out functionId); - } - - gtsContext.SetRawBase(block, operation, baseLo, baseHi, constOffset); } if (gtsContext.TryGetFunctionId(operation, isMultiTarget: true, targetCbs, out functionId)) @@ -894,7 +939,8 @@ private static bool TryGenerateRawStorageOp( ResourceManager resourceManager, IGpuAccessor gpuAccessor, Operation operation, - out int functionId) + out int functionId, + int vectorCount = 1) { functionId = 0; @@ -1040,7 +1086,18 @@ Operand Below(Operand high, Operand low) Operand resultValue = null; - if (operation.Inst == Instruction.Load) + if (operation.Inst == Instruction.Load && vectorCount > 1) + { + resultValue = context.Add( + Instruction.Load, + StorageKind.GlobalMemoryVector, + Local(), + hostLow, + hostHigh, + Const(vectorCount), + Const(resourceManager.GetRawVectorMemory())); + } + else if (operation.Inst == Instruction.Load) { resultValue = context.Load(StorageKind.GlobalMemory, hostLow, hostHigh); @@ -1105,315 +1162,35 @@ Operand Below(Operand high, Operand low) if (returnsValue) { - context.Return(Const(0)); - } - else - { - context.Return(); - } - - Function function = new( - ControlFlowGraph.Create(context.GetOperations()).Blocks, - GetFunctionName(operation, isMultiTarget: true, []) + "Raw", - returnsValue, - inArgumentsCount, - 0); - - functionId = gtsContext.AddFunction(operation, isMultiTarget: true, [], function); - - return true; - } - - private static bool TryGenerateRawSameBaseStorageOp( - GtsContext gtsContext, - ResourceManager resourceManager, - Operation operation, - int lastBinding, - int fallbackFunctionId, - out int functionId) - { - functionId = 0; - - if (!resourceManager.TryGetStorageBufferBinding(GmemMapCbSlot, 0, write: false, out int mapBinding)) - { - return false; - } - - bool isSmall = IsSmallInt(operation.StorageKind); - int inArgumentsCount = operation.Inst == Instruction.Store ? 3 : 2; - - EmitterContext context = new(); - - Operand addrLow = Argument(0); - Operand addrHigh = Argument(1); - - Operand MapWord(Operand entry, int word) - { - return context.Load(StorageKind.StorageBuffer, mapBinding, Const(0), context.IAdd(entry, Const(word))); - } - - Operand last = context.Load(StorageKind.LocalMemory, lastBinding); - Operand found = context.IMultiply(last, Const(GmemMapEntryWords)); - Operand startsAtOrBelow = context.BitwiseOr( - context.ICompareLessUnsigned(MapWord(found, 1), addrHigh), - context.BitwiseAnd( - context.ICompareEqual(MapWord(found, 1), addrHigh), - context.ICompareLessOrEqualUnsigned(MapWord(found, 0), addrLow))); - Operand endsAbove = context.BitwiseOr( - context.ICompareLessUnsigned(addrHigh, MapWord(found, 3)), - context.BitwiseAnd( - context.ICompareEqual(addrHigh, MapWord(found, 3)), - context.ICompareLessUnsigned(addrLow, MapWord(found, 2)))); - Operand lblFallback = Label(); - context.BranchIfFalse(lblFallback, context.BitwiseAnd(startsAtOrBelow, endsAbove)); - - Operand wordLow = isSmall ? context.BitwiseAnd(addrLow, Const(~3)) : addrLow; - Operand hostLow = context.IAdd(wordLow, MapWord(found, 4)); - Operand carry = context.ConditionalSelect(context.ICompareLessUnsigned(hostLow, wordLow), Const(1), Const(0)); - Operand hostHigh = context.IAdd(context.IAdd(addrHigh, MapWord(found, 5)), carry); - - Operand resultValue = null; - - if (operation.Inst == Instruction.Load) - { - resultValue = context.Load(StorageKind.GlobalMemory, hostLow, hostHigh); - - if (isSmall) + // A miss must preserve the scalar semantics: every word reads + // zero, including the followers stored in gmem_vec. + for (int word = 0; word < vectorCount - 1; word++) { - Operand bitOffset = HelperFunctionManager.GetBitOffset(context, addrLow); - - resultValue = operation.StorageKind switch - { - StorageKind.GlobalMemoryS8 => context.BitfieldExtractS32(context.ShiftRightS32(resultValue, bitOffset), Const(0), Const(8)), - StorageKind.GlobalMemoryS16 => context.BitfieldExtractS32(context.ShiftRightS32(resultValue, bitOffset), Const(0), Const(16)), - StorageKind.GlobalMemoryU8 => context.BitwiseAnd(context.ShiftRightU32(resultValue, bitOffset), Const(byte.MaxValue)), - _ => context.BitwiseAnd(context.ShiftRightU32(resultValue, bitOffset), Const(ushort.MaxValue)), - }; + context.Store( + StorageKind.LocalMemory, + resourceManager.GetRawVectorMemory(), + Const(word), + Const(0)); } - } - else - { - context.Store(StorageKind.GlobalMemory, hostLow, hostHigh, Argument(2)); - } - - if (resultValue != null) - { - context.Return(resultValue); - } - else - { - context.Return(); - } - context.MarkLabel(lblFallback); - - bool returnsValue = operation.Dest != null; - - if (returnsValue) - { - context.Return(context.Call(fallbackFunctionId, true, addrLow, addrHigh)); + context.Return(Const(0)); } else { - context.Call(fallbackFunctionId, false, addrLow, addrHigh, Argument(2)); context.Return(); } Function function = new( ControlFlowGraph.Create(context.GetOperations()).Blocks, - GetFunctionName(operation, isMultiTarget: true, []) + "RawSameBase", + GetFunctionName(operation, isMultiTarget: true, []) + + (vectorCount > 1 ? $"RawV{vectorCount}" : "Raw"), returnsValue, inArgumentsCount, 0); - functionId = gtsContext.AddRawSameBaseFunction(operation, function); - - return true; - } - - private static void DecomposeRawAddress( - BasicBlock block, - Operand addrLow, - Operand addrHigh, - out Operand baseLo, - out Operand baseHi, - out int constOffset) - { - addrLow = Utils.FindLastOperation(addrLow, block); - addrHigh = Utils.FindLastOperation(addrHigh, block); - - if (addrLow.AsgOp is Operation addLo && - addLo.Inst == Instruction.Add && - addrHigh.AsgOp is Operation addHi && - addHi.Inst == Instruction.Add) - { - Operand lo0 = addLo.GetSource(0); - Operand lo1 = addLo.GetSource(1); - Operand bLo = null; - Operand cOp = null; - - if (lo0.Type == OperandType.LocalVariable && lo1.Type == OperandType.Constant) - { - bLo = lo0; - cOp = lo1; - } - else if (lo0.Type == OperandType.Constant && lo1.Type == OperandType.LocalVariable) - { - bLo = lo1; - cOp = lo0; - } - - if (bLo != null) - { - Operand hi0 = Utils.FindLastOperation(addHi.GetSource(0), block); - Operand hi1 = Utils.FindLastOperation(addHi.GetSource(1), block); - Operation hi0Op = hi0.AsgOp as Operation; - Operation hi1Op = hi1.AsgOp as Operation; - - if (hi1Op?.Inst == Instruction.ConditionalSelect && hi0Op?.Inst != Instruction.ConditionalSelect) - { - baseLo = Utils.FindLastOperation(bLo, block); - baseHi = hi0; - constOffset = UnwrapScaledIndex(block, ref baseLo, ref baseHi) ? 0 : cOp.Value; - return; - } - else if (hi0Op?.Inst == Instruction.ConditionalSelect && hi1Op?.Inst != Instruction.ConditionalSelect) - { - baseLo = Utils.FindLastOperation(bLo, block); - baseHi = hi1; - constOffset = UnwrapScaledIndex(block, ref baseLo, ref baseHi) ? 0 : cOp.Value; - return; - } - } - } - - baseLo = addrLow; - baseHi = addrHigh; - constOffset = 0; - UnwrapScaledIndex(block, ref baseLo, ref baseHi); - } - - private static bool UnwrapScaledIndex(BasicBlock block, ref Operand baseLo, ref Operand baseHi) - { - if (baseLo.AsgOp is not Operation addLo || addLo.Inst != Instruction.Add || - baseHi.AsgOp is not Operation addHi || addHi.Inst != Instruction.Add) - { - return false; - } - - Operand hi0 = Utils.FindLastOperation(addHi.GetSource(0), block); - Operand hi1 = Utils.FindLastOperation(addHi.GetSource(1), block); - Operation hi0Op = hi0.AsgOp as Operation; - Operation hi1Op = hi1.AsgOp as Operation; - Operation hiSumOp = null; - - if (hi1Op?.Inst == Instruction.ConditionalSelect && hi0Op?.Inst == Instruction.Add) - { - hiSumOp = hi0Op; - } - else if (hi0Op?.Inst == Instruction.ConditionalSelect && hi1Op?.Inst == Instruction.Add) - { - hiSumOp = hi1Op; - } - - if (hiSumOp == null) - { - return false; - } - - Operand lo0 = Utils.FindLastOperation(addLo.GetSource(0), block); - Operand lo1 = Utils.FindLastOperation(addLo.GetSource(1), block); - Operand hs0 = Utils.FindLastOperation(hiSumOp.GetSource(0), block); - Operand hs1 = Utils.FindLastOperation(hiSumOp.GetSource(1), block); - - static bool IsMulLo(Operand op) => (op.AsgOp as Operation)?.Inst is Instruction.Multiply or Instruction.ShiftLeft; - static bool IsMulHi(Operand op) => (op.AsgOp as Operation)?.Inst is Instruction.MultiplyHighU32 or Instruction.MultiplyHighS32 or Instruction.ShiftRightU32; - - Operand arrLo = null; - Operand arrHi = null; - - if (IsMulLo(lo0) && !IsMulLo(lo1)) - { - arrLo = lo1; - } - else if (IsMulLo(lo1) && !IsMulLo(lo0)) - { - arrLo = lo0; - } - - if (IsMulHi(hs0) && !IsMulHi(hs1)) - { - arrHi = hs1; - } - else if (IsMulHi(hs1) && !IsMulHi(hs0)) - { - arrHi = hs0; - } - - if (arrLo != null && arrHi != null) - { - baseLo = arrLo; - baseHi = arrHi; - return true; - } - - return false; - } - - private static bool SameAddressOperand(BasicBlock block, Operand a, Operand b, int depth = 0) - { - if (a == null || b == null) - { - return false; - } - - a = Utils.FindLastOperation(a, block); - b = Utils.FindLastOperation(b, block); - - if (a == b) - { - return true; - } - - if (a.Type == OperandType.Constant && b.Type == OperandType.Constant) - { - return a.Value == b.Value; - } - - if (depth >= 4 || a.AsgOp is not Operation opA || b.AsgOp is not Operation opB) - { - return false; - } - - if (opA.Inst != opB.Inst || opA.StorageKind != opB.StorageKind || opA.SourcesCount != opB.SourcesCount) - { - return false; - } - - switch (opA.Inst) - { - case Instruction.Add: - case Instruction.Multiply: - case Instruction.MultiplyHighS32: - case Instruction.MultiplyHighU32: - case Instruction.ShiftLeft: - case Instruction.ShiftRightU32: - case Instruction.ConditionalSelect: - case Instruction.CompareEqual: - case Instruction.CompareLessU32: - case Instruction.CompareGreaterU32: - break; - default: - return false; - } - - for (int i = 0; i < opA.SourcesCount; i++) - { - if (!SameAddressOperand(block, opA.GetSource(i), opB.GetSource(i), depth + 1)) - { - return false; - } - } + functionId = vectorCount > 1 + ? gtsContext.AddRawVectorFunction(vectorCount, function) + : gtsContext.AddFunction(operation, isMultiTarget: true, [], function); return true; } diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs index 2fbad681..deb9ce19 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs @@ -165,6 +165,21 @@ public int AddRawLastEntryMemory() return Properties.AddLocalMemory(new MemoryDefinition($"gmem_last_{Properties.LocalMemories.Count}", AggregateType.U32, 1, zeroInit: true)); } + // Words 1..3 returned by one raw vector load. Followers consume them + // immediately, before another memory access can overwrite the array. + private int _rawVectorMemoryId = -1; + + public int GetRawVectorMemory() + { + if (_rawVectorMemoryId < 0) + { + _rawVectorMemoryId = Properties.AddLocalMemory( + new MemoryDefinition("gmem_vec", AggregateType.Array | AggregateType.U32, 3)); + } + + return _rawVectorMemoryId; + } + private int AddMemoryDefinition(string name, AggregateType type, int arrayLength = 1) { return Properties.AddLocalMemory(new MemoryDefinition(name, type, arrayLength)); From d83d66abc3d358f2e8154845214e61ec91cf2b97 Mon Sep 17 00:00:00 2001 From: seedds Date: Wed, 11 Nov 2026 21:33:41 +0100 Subject: [PATCH 6/6] shader: bound wide loads to one pool --- dm/gfx/tests/test_dmsh_raw_reuse.py | 2 ++ .../Optimizations/GlobalToStorage.cs | 22 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/dm/gfx/tests/test_dmsh_raw_reuse.py b/dm/gfx/tests/test_dmsh_raw_reuse.py index 30b644dd..edb79746 100644 --- a/dm/gfx/tests/test_dmsh_raw_reuse.py +++ b/dm/gfx/tests/test_dmsh_raw_reuse.py @@ -111,6 +111,8 @@ def test_128_bit_load_uses_one_vector_helper(self): functions = self.functions(disassembly) helper = functions[calls[0]] self.assertIn('OpLoopMerge', helper) + self.assertIn('%int_15', helper, + 'the resolver must check the 128-bit load\'s last byte') self.assertRegex(helper, r'OpLoad %v4uint \S+ Aligned 4') self.assertGreaterEqual( len(re.findall(r'OpStore \S+ %uint_0', helper)), 4, diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs index 470b4438..9a72f2eb 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs @@ -1054,7 +1054,27 @@ Operand Below(Operand high, Operand low) context.Store(StorageKind.LocalMemory, lastBinding, selected); Operand found = context.IMultiply(selected, Const(GmemMapEntryWords)); Operand lblSkip = Label(); - context.BranchIfFalse(lblSkip, Below(MapWord(found, 3), MapWord(found, 2))); + Operand inPool = Below(MapWord(found, 3), MapWord(found, 2)); + + if (vectorCount > 1) + { + // The resolver chose the entry from word zero. Require the + // vector's last byte to remain below its end as well, rather + // than relying on host pool alignment. + Operand lastLow = context.IAdd(addrLow, Const(vectorCount * 4 - 1)); + Operand endCarry = context.ConditionalSelect( + context.ICompareLessUnsigned(lastLow, addrLow), Const(1), Const(0)); + Operand lastHigh = context.IAdd(addrHigh, endCarry); + Operand endLow = MapWord(found, 2); + Operand endHigh = MapWord(found, 3); + inPool = context.BitwiseOr( + context.ICompareLessUnsigned(lastHigh, endHigh), + context.BitwiseAnd( + context.ICompareEqual(lastHigh, endHigh), + context.ICompareLessUnsigned(lastLow, endLow))); + } + + context.BranchIfFalse(lblSkip, inPool); context.MarkLabel(lblChosen); found = context.IMultiply(selected, Const(GmemMapEntryWords));