diff --git a/dm/gfx/tests/test_dmsh_ampere_mesh.py b/dm/gfx/tests/test_dmsh_ampere_mesh.py index 80a1f49b..3fd98009 100644 --- a/dm/gfx/tests/test_dmsh_ampere_mesh.py +++ b/dm/gfx/tests/test_dmsh_ampere_mesh.py @@ -28,9 +28,9 @@ def program(*insts): return bytes(0x80) + b''.join(insts) + inst(*EXIT) # a ze # SPIR-V numbers the checks read OP_ENTRY_POINT, OP_EXECUTION_MODE, OP_DECORATE, OP_VARIABLE, OP_EMIT_MESH_TASKS, OP_SET_MESH_OUTPUTS = 15, 16, 71, 59, 5294, 5295 MODEL_TASK, MODEL_MESH = 5364, 5365 -MODE_LOCAL_SIZE, MODE_OUTPUT_VERTICES, MODE_OUTPUT_PRIMITIVES, MODE_OUTPUT_TRIANGLES = 17, 26, 5270, 5298 +MODE_LOCAL_SIZE, MODE_OUTPUT_POINTS, MODE_OUTPUT_VERTICES, MODE_OUTPUT_PRIMITIVES, MODE_OUTPUT_LINES, MODE_OUTPUT_TRIANGLES = 17, 27, 26, 5270, 5269, 5298 DEC_SPEC_ID, DEC_BUILTIN, DEC_LOCATION, DEC_PER_PRIMITIVE = 1, 11, 30, 5271 -BUILTIN_PRIMITIVE_TRIANGLE_INDICES = 5296 +BUILTIN_POINT_SIZE, BUILTIN_PRIMITIVE_POINT_INDICES, BUILTIN_PRIMITIVE_LINE_INDICES, BUILTIN_PRIMITIVE_TRIANGLE_INDICES = 1, 5294, 5295, 5296 SC_TASK_PAYLOAD = 5402 @@ -86,6 +86,17 @@ def test_task(self): self.assertEqual(ops[OP_EMIT_MESH_TASKS][0][3], payload[0][1]) # the payload goes with the launch self.assertEqual(res['stage'], 'ts') + def test_task_reads_its_own_payload(self): + # 0x927 / 0xb8 writes a record, then 0x923 / 0xb8 reads the same word in another thread. The loaded value becomes the mesh-workgroup count, so it survives optimization. + code = program(stage(STAGE_LOAD, HEADER, rd=0), stage(STAGE_STORE, TASK_WRITE, rb=0, imm=0x10), + stage(STAGE_LOAD, TASK_WRITE, rd=1, imm=0x10), stage(STAGE_STORE, TASK_WRITE, rb=1, imm=4)) + (_, ops), _ = self.translate(code, stage='ts', local=[32, 1, 1], mesh=[0, 0, 64]) + self.assertEqual([e[0] for e in ops[OP_ENTRY_POINT]], [MODEL_TASK]) + self.assertEqual(len(ops[OP_EMIT_MESH_TASKS]), 1) + + def test_task_refuses_mesh_topology(self): + self.refused(self.TASK, 'task program with mesh output', stage='ts', local=[32, 1, 1], mesh=[0, 0, 64, -5, 1]) + # A mesh program: its index (task memory word 0), a payload word by it, the four position planes and the generic planes of vertex 0, one per-primitive value of # primitive 0, triangle 0's indices (bytes 4..6) and the primitive count (byte 3) MESH = program(stage(STAGE_LOAD, TASK_READ, rd=1), stage(STAGE_LOAD, TASK_READ, rd=2, ra=1, imm=0x10), @@ -121,6 +132,27 @@ def test_mesh_without_payload(self): # a mesh program run without a task reads stage='ms', local=[32, 1, 1], mesh=[32, 16, 128, 0, 1, 2, 3]) self.assertFalse([v for v in ops[OP_VARIABLE] if v[2] == SC_TASK_PAYLOAD]) + def test_point_mesh_with_point_size(self): + code = program(*[stage(STAGE_STORE, PLANES, rb=1, imm=0x80 * p) for p in range(5)], + stage(STAGE_STORE, INDICES, rb=RZ, imm=4), stage(STAGE_STORE, COUNTS, rb=1, imm=3)) + (_, ops), _ = self.translate(code, stage='ms', local=[32, 1, 1], mesh=[32, 32, 64, -5, 1, -4, 0, 1, 2, 3]) + ep = ops[OP_ENTRY_POINT][0] + modes = {m[1]: m[2:] for m in ops[OP_EXECUTION_MODE] if m[0] == ep[1]} + self.assertIn(MODE_OUTPUT_POINTS, modes) + decos = [d for d in ops[OP_DECORATE] if d[1] == DEC_BUILTIN] + self.assertTrue(any(d[2] == BUILTIN_PRIMITIVE_POINT_INDICES for d in decos)) + self.assertTrue(any(d[2] == DEC_BUILTIN and d[3] == BUILTIN_POINT_SIZE for d in ops.get(72, []))) + + def test_line_mesh(self): + code = program(*[stage(STAGE_STORE, PLANES, rb=1, imm=0x80 * p) for p in range(4)], + stage(STAGE_STORE, INDICES, rb=RZ, imm=4), stage(STAGE_STORE, INDICES, rb=1, imm=5), + stage(STAGE_STORE, COUNTS, rb=1, imm=3)) + (_, ops), _ = self.translate(code, stage='ms', local=[32, 1, 1], mesh=[32, 32, 64, -5, 2, 0, 1, 2, 3]) + ep = ops[OP_ENTRY_POINT][0] + modes = {m[1]: m[2:] for m in ops[OP_EXECUTION_MODE] if m[0] == ep[1]} + self.assertIn(MODE_OUTPUT_LINES, modes) + self.assertTrue(any(d[1] == DEC_BUILTIN and d[2] == BUILTIN_PRIMITIVE_LINE_INDICES for d in ops[OP_DECORATE])) + def test_refused_mode(self): self.refused(program(stage(STAGE_STORE, 0x9800, rb=0)), 'stage memory mode 0x9800', stage='ms', local=[32, 1, 1], mesh=[32, 16, 128, 0, 1, 2, 3]) diff --git a/dm/gfx/tests/test_dmsh_raw_reuse.py b/dm/gfx/tests/test_dmsh_raw_reuse.py new file mode 100644 index 00000000..edb79746 --- /dev/null +++ b/dm/gfx/tests/test_dmsh_raw_reuse.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Raw-global lookups use the current hint ABI and vectorize wide loads. + +The first access tries the tagged 2 MiB-chunk hint before a looped table search. +Later accesses first check the entry cached for this invocation. LDG.64 and +LDG.128 resolve once and perform one aligned vector load, including zeroing all +follower words when the address misses every pool. +""" +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from test_dmsh_ampere_mem import EXIT, Translated, inst # noqa: E402 + +TOOLS = Path(__file__).resolve().parents[1] / 'tools' +STORE = (0x000fe8000000c800, 0x0000080609008388) +LDG = (0x000ea200081e890f, 0x0000000405067981) +VEC128 = ((LDG[0] & ~(7 << 9)) | (6 << 9), + (LDG[1] & ~0xff0000) | (12 << 16)) + + +def waiting(instruction): + """Make the instruction wait for every scoreboard dependency.""" + return (instruction[0] | 0x3f << 52, instruction[1]) + + +def s2r_tid(register): + """Read SR_TID.X into *register* using the fixture's known SM86 form.""" + return (0x000e260000002100, + 0x0000000000007919 | register << 16) + + +@unittest.skipUnless(os.environ.get('DMSH_TEST_XLATE'), + 'set DMSH_TEST_XLATE to the offline translator') +@unittest.skipUnless(shutil.which('spirv-dis'), 'needs spirv-dis') +class RawReuse(Translated): + def spv_dis(self, code): + sys.path.insert(0, str(TOOLS)) + import dmsh_geometry + + with tempfile.TemporaryDirectory(prefix='raw_reuse_') as directory: + work = Path(directory) + (work / 'k.bin').write_bytes( + b''.join(inst(*waiting(op)) for op in code) + inst(*EXIT)) + job = [dict(key='k', stage='cs', arch='sm86', + code=str(work / 'k.bin'), tex={}, img={}, attr={}, + fout={}, local=[32, 1, 1], debug=True, + nvn2_cs=[0, 0, 3, 0x40, 0], shared=0x1000)] + (work / 'jobs.json').write_text(json.dumps(job)) + run = subprocess.run( + [os.environ['DMSH_TEST_XLATE'], str(work / 'jobs.json'), + directory], capture_output=True, text=True) + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + dmsh_geometry.flatten_sets(work / 'k.spv') + if shutil.which('spirv-val'): + valid = subprocess.run( + ['spirv-val', '--target-env', 'vulkan1.3', + str(work / 'k.spv')], capture_output=True, text=True) + self.assertEqual(valid.returncode, 0, + valid.stdout + valid.stderr) + return subprocess.run( + ['spirv-dis', str(work / 'k.spv')], capture_output=True, + text=True, check=True).stdout + + @staticmethod + def functions(disassembly): + return {match.group(1): match.group(0) for match in re.finditer( + r'(%\w+) = OpFunction %int None .*?OpFunctionEnd', + disassembly, re.S)} + + @staticmethod + def main_function(disassembly): + return re.search( + r'%\w+ = OpFunction %void None .*?OpFunctionEnd', + disassembly, re.S).group(0) + + def test_loads_share_one_zero_initialized_last_entry(self): + second = (LDG[0], LDG[1] | 4 << 40) + disassembly = self.spv_dis( + (s2r_tid(9), s2r_tid(5), LDG, STORE, second, STORE)) + self.assertEqual(len(re.findall( + r'OpVariable \S+ Private %\w+', disassembly)), 1) + main = self.main_function(disassembly) + calls = re.findall(r'OpFunctionCall %int (%\w+)', main) + self.assertEqual(len(calls), 2) + self.assertEqual(calls[0], calls[1]) + full = self.functions(disassembly)[calls[0]] + self.assertIn('OpLoopMerge', full) + self.assertIn('OpConstant %int 8191', disassembly) + self.assertRegex(full, r'OpShiftRightLogical %int \S+ %int_21') + self.assertRegex(full, r'OpBitwiseAnd %int \S+ %int_8191') + self.assertRegex(full, r'OpShiftRightLogical %int \S+ %int_13') + + def test_128_bit_load_uses_one_vector_helper(self): + stores = tuple((STORE[0], (STORE[1] & ~0xff000000) | + (register << 24) | (offset << 40)) + for register, offset in + ((12, 0), (13, 4), (14, 16), (15, 20))) + disassembly = self.spv_dis( + (s2r_tid(9), s2r_tid(5), VEC128, *stores)) + main = self.main_function(disassembly) + calls = re.findall(r'OpFunctionCall %int (%\w+)', main) + self.assertEqual(len(calls), 1) + functions = self.functions(disassembly) + helper = functions[calls[0]] + self.assertIn('OpLoopMerge', helper) + self.assertIn('%int_15', helper, + 'the resolver must check the 128-bit load\'s last byte') + self.assertRegex(helper, r'OpLoad %v4uint \S+ Aligned 4') + self.assertGreaterEqual( + len(re.findall(r'OpStore \S+ %uint_0', helper)), 4, + 'a miss must zero all three follower words and the last-entry cache') + + def test_adjacent_vectors_do_not_share_follower_words(self): + first = (VEC128[0] & ~0xf, VEC128[1]) + second = (first[0], + (first[1] & ~(0xff << 16) & ~(0xffffff << 40)) | + (16 << 16) | (0x40 << 40)) + keep_first = (STORE[0], (STORE[1] & ~0xff000000) | (12 << 24)) + keep_second = (STORE[0], + (STORE[1] & ~0xff000000 & ~(0xffffff << 40)) | + (18 << 24) | (4 << 40)) + disassembly = self.spv_dis( + (s2r_tid(9), s2r_tid(5), first, second, + keep_first, keep_second)) + calls = re.findall( + r'OpFunctionCall %int (%\w+)', self.main_function(disassembly)) + self.assertEqual(len(calls), 2) + functions = self.functions(disassembly) + self.assertEqual(sum('OpLoad %v4uint' in functions[call] + for call in set(calls)), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs index a7b82e74..6230f6c0 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs @@ -331,7 +331,7 @@ private static void DeclareMemories(CodeGenContext context, IEnumerable context.Constant(tU32, v); uint threads = (uint)Math.Max(1, context.Definitions.ComputeLocalSizeX * context.Definitions.ComputeLocalSizeY * context.Definitions.ComputeLocalSizeZ); List io = new(context.GetMainInterface()); @@ -55,9 +56,14 @@ public static void Generate(CodeGenContext context, MeshShaderInfo mesh) if (mesh.IsMesh) { - SpvInstruction perVertex = context.TypeStruct(true, tVec4); + bool pointSize = mesh.Planes.Contains(MeshShaderInfo.PointSizePlane); + SpvInstruction perVertex = pointSize ? context.TypeStruct(true, tVec4, tF32) : context.TypeStruct(true, tVec4); context.Decorate(perVertex, Decoration.Block); context.MemberDecorate(perVertex, 0, Decoration.BuiltIn, (SpvLiteralInteger)BuiltIn.Position); + if (pointSize) + { + context.MemberDecorate(perVertex, 1, Decoration.BuiltIn, (SpvLiteralInteger)BuiltIn.PointSize); + } posVar = context.Variable(context.TypePointer(StorageClass.Output, context.TypeArray(perVertex, U((uint)mesh.MaxVertices))), StorageClass.Output); context.Name(posVar, "mesh_position"); context.AddGlobalVariable(posVar); @@ -84,8 +90,15 @@ public static void Generate(CodeGenContext context, MeshShaderInfo mesh) io.Add(v); } - idxVar = context.Variable(context.TypePointer(StorageClass.Output, context.TypeArray(tUVec3, U((uint)mesh.MaxPrimitives))), StorageClass.Output); - context.Decorate(idxVar, Decoration.BuiltIn, (SpvLiteralInteger)BuiltIn.PrimitiveTriangleIndicesEXT); + SpvInstruction indexType = mesh.PrimitiveVertices switch { 1 => tU32, 2 => tUVec2, _ => tUVec3 }; + BuiltIn indexBuiltIn = mesh.PrimitiveVertices switch + { + 1 => BuiltIn.PrimitivePointIndicesEXT, + 2 => BuiltIn.PrimitiveLineIndicesEXT, + _ => BuiltIn.PrimitiveTriangleIndicesEXT, + }; + idxVar = context.Variable(context.TypePointer(StorageClass.Output, context.TypeArray(indexType, U((uint)mesh.MaxPrimitives))), StorageClass.Output); + context.Decorate(idxVar, Decoration.BuiltIn, (SpvLiteralInteger)indexBuiltIn); context.AddGlobalVariable(idxVar); io.Add(idxVar); } @@ -163,7 +176,8 @@ void ForEach(SpvInstruction lidValue, int count, Action then) // position and the generic locations whose components the planes do not all name, and every per-primitive location bool posFull = Enumerable.Range(0, 4).All(c => mesh.Planes.Contains(c)); List vertexInit = generics.Where(kv => !Enumerable.Range(0, 4).All(c => mesh.Planes.Contains(4 + 4 * kv.Key + c))).Select(kv => kv.Value).ToList(); - if (vertexInit.Count != 0 || !posFull) + bool pointSize = mesh.Planes.Contains(MeshShaderInfo.PointSizePlane); + if (vertexInit.Count != 0 || !posFull || pointSize) { ForEach(lid, mesh.MaxVertices, v => { @@ -172,6 +186,11 @@ void ForEach(SpvInstruction lidValue, int count, Action then) context.Store(context.AccessChain(pOutVec4, posVar, v, U(0)), init); } + if (pointSize) + { + context.Store(context.AccessChain(pOutF32, posVar, v, U(1)), context.Constant(tF32, 1f)); + } + foreach (SpvInstruction g in vertexInit) { context.Store(context.AccessChain(pOutVec4, g, v), init); @@ -215,7 +234,8 @@ void ForEach(SpvInstruction lidValue, int count, Action then) { SpvInstruction prims = context.SharedMemories[mesh.PrimitiveMemoryId]; SpvInstruction primCount = context.Load(tU32, context.AccessChain(pWg, prims, U(3))); - SpvInstruction pOutUVec3 = context.TypePointer(StorageClass.Output, tUVec3); + SpvInstruction indexType = mesh.PrimitiveVertices switch { 1 => tU32, 2 => tUVec2, _ => tUVec3 }; + SpvInstruction pOutIndex = context.TypePointer(StorageClass.Output, indexType); SpvInstruction lastVertex = U((uint)mesh.MaxVertices - 1); SpvInstruction Index(SpvInstruction word) @@ -227,10 +247,23 @@ SpvInstruction Index(SpvInstruction word) ForEach(lid, mesh.MaxPrimitives, t => { - SpvInstruction a = context.IAdd(tU32, context.IMul(tU32, t, U(3)), U(4)); - SpvInstruction tri = context.CompositeConstruct(tUVec3, Index(a), Index(context.IAdd(tU32, a, U(1))), Index(context.IAdd(tU32, a, U(2)))); - tri = context.Select(tUVec3, context.ULessThan(tBool, t, primCount), tri, context.ConstantComposite(tUVec3, U(0), U(0), U(0))); - context.Store(context.AccessChain(pOutUVec3, idxVar, t), tri); + if (mesh.PrimitiveVertices == 3) + { + SpvInstruction a = context.IAdd(tU32, context.IMul(tU32, t, U(3)), U(4)); + SpvInstruction tri = context.CompositeConstruct(tUVec3, Index(a), Index(context.IAdd(tU32, a, U(1))), Index(context.IAdd(tU32, a, U(2)))); + tri = context.Select(tUVec3, context.ULessThan(tBool, t, primCount), tri, context.ConstantComposite(tUVec3, U(0), U(0), U(0))); + context.Store(context.AccessChain(pOutIndex, idxVar, t), tri); + } + else + { + SpvInstruction a = context.IAdd(tU32, context.IMul(tU32, t, U((uint)mesh.PrimitiveVertices)), U(4)); + SpvInstruction indices = mesh.PrimitiveVertices == 1 + ? Index(a) + : context.CompositeConstruct(tUVec2, Index(a), Index(context.IAdd(tU32, a, U(1)))); + SpvInstruction zero = mesh.PrimitiveVertices == 1 ? U(0) : context.ConstantComposite(tUVec2, U(0), U(0)); + indices = context.Select(indexType, context.ULessThan(tBool, t, primCount), indices, zero); + context.Store(context.AccessChain(pOutIndex, idxVar, t), indices); + } }); // The GL depth convention (vk_android_spv.gl_depth's arithmetic for a vertex shader): z' = fma(z, 0.5, w * 0.5) when the constant is true @@ -285,9 +318,11 @@ void Scatter(SpvInstruction w, int chunkWords, int count, int[] planes, Action Scatter(word, mesh.VertexChunkWords, mesh.MaxVertices, mesh.Planes, (v, plane) => { - context.Store(plane < 4 - ? context.AccessChain(pOutF32, posVar, v, U(0), U((uint)plane)) - : context.AccessChain(pOutF32, generics[(plane - 4) >> 2], v, U((uint)(plane & 3))), f); + context.Store(plane == MeshShaderInfo.PointSizePlane + ? context.AccessChain(pOutF32, posVar, v, U(1)) + : plane < 4 + ? context.AccessChain(pOutF32, posVar, v, U(0), U((uint)plane)) + : context.AccessChain(pOutF32, generics[(plane - 4) >> 2], v, U((uint)(plane & 3))), f); if (zw != null && plane is 2 or 3) { @@ -326,7 +361,12 @@ void Scatter(SpvInstruction w, int chunkWords, int count, int[] planes, Action ExecutionMode.OutputPoints, + 2 => ExecutionMode.OutputLinesEXT, + _ => ExecutionMode.OutputTrianglesEXT, + }); } } } diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereAlu.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereAlu.cs index 8b2b57f6..b181718d 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereAlu.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereAlu.cs @@ -35,6 +35,7 @@ public static void RegisterAmpereAlu(AmpereAluRegistrar alu, AmpereRegistrar x) alu(0x107, InstName.F2f, AFrnd); alu(0x104, InstName.F2f, AF2f); alu(0x03e, InstName.F2f, AF2fp); + alu(0x043, InstName.F2i, AF2ip); alu(0x030, InstName.Hadd2, AHadd2); alu(0x032, InstName.Hmul2, AHmul2); alu(0x031, InstName.Hfma2, AHfma2); @@ -433,6 +434,36 @@ public static void AF2fp(EmitterContext context) ASet(context, x, context.PackHalf2x16(AFloat(context, b), AFloat(context, a)), AUniform(context)); } + // F2IP.U8 Rd, a, b: a and b to saturated bytes, a in byte 1 and b in + // byte 0, like F2FP's halves; bits 78..79 select F2I rounding. + public static void AF2ip(EmitterContext context) + { + AmpereOp x = new(context.CurrOp); + + if (x.Bits(72, 78) != 4 || x.Rc != RegisterConsts.RegisterZeroIndex) + { + throw AFail(context, "F2IP form"); + } + + (ASrc a, ASrc b) = ASecond(context, x, abs: false); + RoundMode2 mode = (RoundMode2)x.Bits(78, 80); + + Operand Byte(Operand f) + { + f = mode switch + { + RoundMode2.Round => context.FPRound(f, Instruction.FP32), + RoundMode2.Floor => context.FPFloor(f, Instruction.FP32), + RoundMode2.Ceil => context.FPCeiling(f, Instruction.FP32), + _ => context.FPTruncate(f, Instruction.FP32), + }; + + return context.FP32ConvertToU32(context.FPMinimum(context.FPMaximum(f, ConstF(0f)), ConstF(255f))); + } + + ASet(context, x, context.BitwiseOr(context.ShiftLeft(Byte(AFloat(context, a)), Const(8)), Byte(AFloat(context, b))), AUniform(context)); + } + // ---- packed halves ---- // A half-pair operand's two lanes after its swizzle (0 as is, 2 low twice, 3 high twice; 1 the high half of a scalar), abs / neg applied. diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMem.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMem.cs index 69db51cc..8fd63fcc 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMem.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMem.cs @@ -256,6 +256,10 @@ public static void ALdg(EmitterContext context) (los[index], his[index]) = AGlobalAddress(context, x, 90, 32, index * 4); } + bool vector = count > 1 && storageKind == StorageKind.GlobalMemory && + x.Rd + count <= RegisterConsts.RegisterZeroIndex; + int group = vector ? ++context.GlobalVectorGroups : 0; + for (int index = 0; index < count; index++) { if (x.Rd + index >= RegisterConsts.RegisterZeroIndex) @@ -263,7 +267,16 @@ public static void ALdg(EmitterContext context) break; } - context.Copy(Register(x.Rd + index, RegisterType.Gpr), context.Load(storageKind, los[index], his[index])); + Operand value = context.Load(storageKind, los[index], his[index]); + + if (vector && value.AsgOp is Operation load) + { + load.GlobalVectorCount = count; + load.GlobalVectorWord = index; + load.GlobalVectorGroup = group; + } + + context.Copy(Register(x.Rd + index, RegisterType.Gpr), value); } if (lblSkip != null) diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMesh.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMesh.cs index 5a575444..488c27ee 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMesh.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitAmpereMesh.cs @@ -45,6 +45,7 @@ private enum AStageWindow None, Header, TaskMemoryRead, + TaskMemoryOwnRead, TaskMemoryWrite, VertexPlanes, PrimitiveIndices, @@ -82,7 +83,7 @@ public static void AStageMemory(EmitterContext context) int offset = x.Bits(40, 64); AStageWindow window = load - ? mode switch { 0x7800 => AStageWindow.Header, 0x3800 => AStageWindow.TaskMemoryRead, _ => AStageWindow.None } + ? mode switch { 0x7800 => AStageWindow.Header, 0x3800 => AStageWindow.TaskMemoryRead, 0xb800 => AStageWindow.TaskMemoryOwnRead, _ => AStageWindow.None } : mode switch { 0xb800 => AStageWindow.TaskMemoryWrite, 0xf800 => AStageWindow.VertexPlanes, 0x8000 => AStageWindow.PrimitiveIndices, 0x8800 => AStageWindow.PrimitiveHeader, _ => AStageWindow.None }; if (window == AStageWindow.None) @@ -95,7 +96,7 @@ public static void AStageMemory(EmitterContext context) throw AFail(context, "stage memory form"); } - bool taskWindow = window is AStageWindow.TaskMemoryWrite; + bool taskWindow = window is AStageWindow.TaskMemoryOwnRead or AStageWindow.TaskMemoryWrite; bool meshWindow = window is AStageWindow.TaskMemoryRead or AStageWindow.VertexPlanes or AStageWindow.PrimitiveIndices or AStageWindow.PrimitiveHeader; if ((taskWindow && !mesh.IsTask) || (meshWindow && !mesh.IsMesh)) @@ -142,6 +143,17 @@ public static void AStageMemory(EmitterContext context) ASet(context, x, x.Ra == RegisterConsts.RegisterZeroIndex ? loaded : context.ConditionalSelect(context.ICompareEqual(word, Const(0)), index, loaded)); } + return; + + case AStageWindow.TaskMemoryOwnRead: + // A task may read back the payload it wrote through 0x927 / 0xb800. Ocarina of Time's 5d3a096025e375a1 writes records in one thread and + // consumes them in another; this is the same task payload memory, not the mesh workgroup's TaskMemoryRead view. + if (x.Rd != RegisterConsts.RegisterZeroIndex) + { + Operand word = context.ShiftRightU32(context.BitwiseAnd(address, Const(mesh.PayloadWords * 4 - 1)), Const(2)); + ASet(context, x, context.Load(StorageKind.SharedMemory, mesh.PayloadMemoryId, word)); + } + return; } diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs index 06a91796..d04245d3 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs @@ -371,6 +371,9 @@ private static void EmitLdg( (_, Operand addrHigh) = Get40BitsAddress(context, new Register(ra, RegisterType.Gpr), extended, offset); Operand srcA = context.Copy(new Operand(new Register(ra, RegisterType.Gpr))); + bool vector = count > 1 && storageKind == StorageKind.GlobalMemory && + rd + count <= RegisterConsts.RegisterZeroIndex; + int group = vector ? ++context.GlobalVectorGroups : 0; for (int index = 0; index < count; index++) { @@ -383,6 +386,13 @@ private static void EmitLdg( Operand value = context.Load(storageKind, context.IAdd(srcA, Const(offset + index * 4)), addrHigh); + if (vector && value.AsgOp is Operation load) + { + load.GlobalVectorCount = count; + load.GlobalVectorWord = index; + load.GlobalVectorGroup = group; + } + context.Copy(Register(dest), value); } } diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/Operation.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/Operation.cs index dc2f56e7..092d903e 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/Operation.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/Operation.cs @@ -10,6 +10,12 @@ class Operation : INode public bool ForcePrecise { get; set; } + // Words emitted from one LDG.64/LDG.128. The group distinguishes + // adjacent vector loads with the same width. + public int GlobalVectorCount { get; set; } + public int GlobalVectorWord { get; set; } + public int GlobalVectorGroup { get; set; } + private Operand[] _dests; public Operand Dest diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/StorageKind.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/StorageKind.cs index 2b39e85e..0e70f35a 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/StorageKind.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/StorageKind.cs @@ -18,6 +18,7 @@ enum StorageKind GlobalMemoryS16, // TODO: Remove this and store type as a field on the Operation class itself. GlobalMemoryU8, // TODO: Remove this and store type as a field on the Operation class itself. GlobalMemoryU16, // TODO: Remove this and store type as a field on the Operation class itself. + GlobalMemoryVector, // nxnative: a raw vector load at a host GPU address: low, high, word count, and local memory for words 1 and up. } static class StorageKindExtensions diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MemoryDefinition.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MemoryDefinition.cs index 3ea69fde..537c0929 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MemoryDefinition.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MemoryDefinition.cs @@ -8,11 +8,15 @@ readonly struct MemoryDefinition public AggregateType Type { get; } public int ArrayLength { get; } - public MemoryDefinition(string name, AggregateType type, int arrayLength = 1) + // nxnative: the memory starts at zero in each invocation (a SPIR-V null initializer), for state a shader reads before it writes (GlobalToStorage.TryGenerateRawStorageOp's last entry) + public bool ZeroInit { get; } + + public MemoryDefinition(string name, AggregateType type, int arrayLength = 1, bool zeroInit = false) { Name = name; Type = type; ArrayLength = arrayLength; + ZeroInit = zeroInit; } } } diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MeshShaderInfo.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MeshShaderInfo.cs index b83c875c..1b911f6c 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MeshShaderInfo.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/MeshShaderInfo.cs @@ -16,8 +16,9 @@ namespace Ryujinx.Graphics.Shader.StructuredIr // (t % 32) x 4). Its stores go straight into the mesh outputs (CodeGenContext.MeshPlaneStore); the memory itself is one word nothing reads // the primitive area, mesh: byte 3 the primitive count, 4 + 3 t ... triangle t's three vertex indices. One word per byte (each store a whole word, so no // byte merging between invocations), PrimitiveWords words, addresses masked - // The layout (IGpuAccessor.QueryMeshShaderLayout, the job's "mesh"): max vertices, max primitives, payload bytes, the per-vertex plane codes in the order of the - // planes (the SPH's output map: 0-3 position x y z w, 4 + 4 g + c generic g component c), then optionally -1 and the per-primitive value codes (same numbering, + // The layout (IGpuAccessor.QueryMeshShaderLayout, the job's "mesh"): max vertices, max primitives, payload bytes, optionally -5 and the primitive's vertex count, + // then the per-vertex plane codes in the order of the planes (the SPH's output map: -4 point size, 0-3 position x y z w, 4 + 4 g + c generic g component c), + // then optionally -1 and the per-primitive value codes (same numbering, // generic only; -3 for a value the program writes that no output takes: the count sets the chunk stride, values x 0x80), then optionally -2 and the generic locations to declare as per-vertex outputs although the program writes none of their components (the // pixel shader reads them: they hold (0, 0, 0, 1)) class MeshShaderInfo @@ -26,6 +27,8 @@ class MeshShaderInfo // A per-primitive value no output takes (its stores are dropped) public const int NoOutput = -3; + public const int PointSizePlane = -4; + public const int Topology = -5; public bool IsTask { get; } public bool IsMesh => !IsTask; @@ -33,6 +36,7 @@ class MeshShaderInfo public int MaxPrimitives { get; } public int PayloadBytes { get; } public int PayloadWords { get; } + public int PrimitiveVertices { get; } public int[] Planes { get; } public int[] PrimitivePlanes { get; } public int[] PadLocations { get; } @@ -59,13 +63,23 @@ public MeshShaderInfo(bool isTask, int[] layout) MaxPrimitives = layout[1]; PayloadBytes = layout[2]; - int primSplit = Array.IndexOf(layout, -1, 3); - int padSplit = Array.IndexOf(layout, -2, 3); + int planeStart = 3; + PrimitiveVertices = 3; + + bool hasTopology = layout.Length >= 5 && layout[3] == Topology; + if (hasTopology) + { + PrimitiveVertices = layout[4]; + planeStart = 5; + } + + int primSplit = Array.IndexOf(layout, -1, planeStart); + int padSplit = Array.IndexOf(layout, -2, planeStart); int end = layout.Length; int planeEnd = primSplit >= 0 ? primSplit : padSplit >= 0 ? padSplit : end; int primEnd = padSplit >= 0 ? padSplit : end; - Planes = layout[3..planeEnd]; + Planes = layout[planeStart..planeEnd]; PrimitivePlanes = primSplit >= 0 ? layout[(primSplit + 1)..primEnd] : Array.Empty(); PadLocations = padSplit >= 0 ? layout[(padSplit + 1)..end] : Array.Empty(); @@ -76,7 +90,8 @@ public MeshShaderInfo(bool isTask, int[] layout) throw new NotSupportedException($"mesh program with {MaxVertices} vertices / {MaxPrimitives} primitives at most (1-256 each)"); } - if (Planes.Any(p => p < 0 || p >= 4 + 4 * 32) || PrimitivePlanes.Any(p => p != NoOutput && (p < 4 || p >= 4 + 4 * 32)) || PadLocations.Any(g => g < 0 || g >= 32) || + if (PrimitiveVertices is < 1 or > 3 || Planes.Any(p => p != PointSizePlane && (p < 0 || p >= 4 + 4 * 32)) || + PrimitivePlanes.Any(p => p != NoOutput && (p < 4 || p >= 4 + 4 * 32)) || PadLocations.Any(g => g < 0 || g >= 32) || Planes.Distinct().Count() != Planes.Length || PrimitivePlanes.Where(p => p != NoOutput).Distinct().Count() != PrimitivePlanes.Count(p => p != NoOutput)) { throw new NotSupportedException($"mesh layout with a plane code out of range or repeated ({string.Join(", ", layout)})"); @@ -87,13 +102,13 @@ public MeshShaderInfo(bool isTask, int[] layout) throw new NotSupportedException($"mesh layout with a generic location both per vertex and per primitive ({string.Join(", ", layout)})"); } } - else if (Planes.Length != 0 || PrimitivePlanes.Length != 0 || PadLocations.Length != 0) + else if (hasTopology || Planes.Length != 0 || PrimitivePlanes.Length != 0 || PadLocations.Length != 0) { throw new NotSupportedException("task program with mesh output planes in its layout"); } PayloadWords = PowerOfTwo(Math.Max(4, (PayloadBytes + 3) / 4)); - PrimitiveWords = PowerOfTwo(4 + 3 * Math.Max(1, MaxPrimitives)); + PrimitiveWords = PowerOfTwo(4 + PrimitiveVertices * Chunks(Math.Max(1, MaxPrimitives)) * 32); } public static int Chunks(int n) diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs index 4152df40..92564001 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs @@ -22,6 +22,10 @@ class EmitterContext public Block CurrBlock { get; set; } public InstOp CurrOp { get; set; } + // Zero means no vector access; each emitted LDG vector gets a distinct + // positive group so follower scans cannot consume another load's word. + public int GlobalVectorGroups { get; set; } + public int OperationsCount => _operations.Count; private readonly struct BrxTarget diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs index 74878631..9a72f2eb 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs @@ -73,15 +73,44 @@ public bool Equals(LsKey other) private readonly List _entries; private readonly Dictionary> _sharedEntries; + private readonly Dictionary<(Instruction, StorageKind), int> _rawLastBindings; + private readonly Dictionary _rawVectorFunctions; private readonly HelperFunctionManager _hfm; public GtsContext(HelperFunctionManager hfm) { _entries = []; _sharedEntries = new Dictionary>(); + _rawLastBindings = new Dictionary<(Instruction, StorageKind), int>(); + _rawVectorFunctions = []; _hfm = hfm; } + public bool TryGetRawVectorFunctionId(int count, out int functionId) + { + return _rawVectorFunctions.TryGetValue(count, out functionId); + } + + public int AddRawVectorFunction(int count, Function function) + { + int functionId = _hfm.AddFunction(function); + _rawVectorFunctions.Add(count, functionId); + return functionId; + } + + public int GetOrAddRawLastBinding(ResourceManager resourceManager, Operation operation) + { + var key = (operation.Inst, operation.StorageKind); + + if (!_rawLastBindings.TryGetValue(key, out int binding)) + { + binding = resourceManager.AddRawLastEntryMemory(); + _rawLastBindings.Add(key, binding); + } + + return binding; + } + public int AddFunction(Operation baseOp, bool isMultiTarget, IReadOnlyList targetCbs, Function function) { int functionId = _hfm.AddFunction(function); @@ -381,15 +410,116 @@ private static LinkedListNode ReplaceGlobalMemoryWithStorage( targetLanguage, block, operation, - out int functionId)) + out int functionId, + out bool raw)) { return null; } + if (raw && + operation.Inst == Instruction.Load && + operation.StorageKind == StorageKind.GlobalMemory && + operation.GlobalVectorCount > 1 && + operation.GlobalVectorWord == 0 && + TryGetRawVectorFunction( + gtsContext, + resourceManager, + gpuAccessor, + operation, + out int vectorFunctionId)) + { + return GenerateRawVectorLoad(resourceManager, node, operation, vectorFunctionId); + } + return GenerateCallStorageOp(node, operation, null, functionId); } } + private static LinkedListNode GenerateRawVectorLoad( + ResourceManager resourceManager, + LinkedListNode node, + Operation operation, + int functionId) + { + int count = operation.GlobalVectorCount; + int vectorMemoryId = resourceManager.GetRawVectorMemory(); + + node = GenerateCallStorageOp(node, operation, null, functionId); + LinkedListNode insertAfter = node; + + for (LinkedListNode next = node.Next; next != null; next = next.Next) + { + if (next.Value is not Operation follower) + { + continue; + } + + if (follower.Inst == Instruction.Load && + follower.StorageKind == StorageKind.GlobalMemory && + follower.GlobalVectorCount == count && + follower.GlobalVectorGroup == operation.GlobalVectorGroup && + follower.GlobalVectorWord > 0) + { + Operand word = Local(); + Operation load = new( + Instruction.Load, + StorageKind.LocalMemory, + word, + Const(vectorMemoryId), + Const(follower.GlobalVectorWord - 1)); + insertAfter = insertAfter.List.AddAfter(insertAfter, load); + + Operand dest = follower.Dest; + follower.TurnIntoCopy(word); + word.UseOps.Remove(follower); + follower.Dest = null; + next.Value = new Operation(Instruction.Copy, dest, word); + + if (follower.GlobalVectorWord == count - 1) + { + break; + } + + continue; + } + + if (follower.GlobalVectorCount != 0 || + follower.StorageKind != StorageKind.None || + follower.Inst == Instruction.Call || + follower.Inst == Instruction.Barrier || + follower.Inst == Instruction.MemoryBarrier || + follower.Inst.IsAtomic()) + { + break; + } + } + + return node; + } + + private static bool TryGetRawVectorFunction( + GtsContext gtsContext, + ResourceManager resourceManager, + IGpuAccessor gpuAccessor, + Operation operation, + out int functionId) + { + int count = operation.GlobalVectorCount; + + if (gtsContext.TryGetRawVectorFunctionId(count, out functionId)) + { + return true; + } + + return TryGenerateRawStorageOp( + gtsContext, + resourceManager, + gpuAccessor, + operation, + out functionId, + count); + } + private static bool CanUseInlineStorageOp(Operation operation, TargetLanguage targetLanguage) { if (operation.StorageKind != StorageKind.GlobalMemory) @@ -589,11 +719,13 @@ private static bool TryGenerateMultiTargetStorageOp( TargetLanguage targetLanguage, BasicBlock block, Operation operation, - out int functionId) + out int functionId, + out bool raw) { Queue phis = new(); HashSet visited = []; List targetCbs = []; + raw = false; Operand globalAddress = operation.GetSource(0); @@ -655,6 +787,7 @@ private static bool TryGenerateMultiTargetStorageOp( if (targetCbs.Count == 0) { + raw = true; gpuAccessor.Log($"Global memory operation \"{operation.Inst}\" traced to no storage buffer: it goes through the memory pool table."); } @@ -783,8 +916,9 @@ private static bool TryGenerateMultiTargetStorageOp( // slot, which no constant buffer of the game has (dmsh_build.py turns it into DMSH_KIND_GMEM_MAP). 1 << GmemMapLog2 entries of GmemMapEntryWords words each, // {start lo, start hi, end lo, end hi, delta lo, delta hi, 0, 0}, sorted by start; entry 0 is empty (start 0, end 0) and unused entries start at ~0 public const int GmemMapCbSlot = 0xff; - private const int GmemMapLog2 = 12; + private const int GmemMapLog2 = 13; private const int GmemMapEntryWords = 8; + private const int GmemHintShift = 21; // nxnative: the write log (dmsh_abi.h DMSH_GMEM_LOG_*), reserved under the same sentinel slot at offset 1: the set of guest pages (address >> 12) the stores and atomics wrote, // as an open-addressing hash set of 1 << GmemLogLog2 words from word 1 (0 = empty), word 0 set when a page found no free slot in GmemLogProbes tries. The host cannot know where @@ -805,7 +939,8 @@ private static bool TryGenerateRawStorageOp( ResourceManager resourceManager, IGpuAccessor gpuAccessor, Operation operation, - out int functionId) + out int functionId, + int vectorCount = 1) { functionId = 0; @@ -853,34 +988,97 @@ Operand MapWord(Operand entry, int word) return context.Load(StorageKind.StorageBuffer, mapBinding, Const(0), context.IAdd(entry, Const(word))); } - // The last entry whose start is at or below the address (entry 0 starts at 0, so there always is one) - Operand index = Const(0); + Operand AtOrBelow(Operand high, Operand low) + { + return context.BitwiseOr( + context.ICompareLessUnsigned(high, addrHigh), + context.BitwiseAnd(context.ICompareEqual(high, addrHigh), context.ICompareLessOrEqualUnsigned(low, addrLow))); + } + + Operand Below(Operand high, Operand low) + { + return context.BitwiseOr( + context.ICompareLessUnsigned(addrHigh, high), + context.BitwiseAnd(context.ICompareEqual(addrHigh, high), context.ICompareLessUnsigned(addrLow, low))); + } + + Operand selected = Local(); + Operand lblFound = Label(); + Operand lblChosen = Label(); + + // Try the entry this helper chose last in this invocation before + // searching the table. Entry zero is empty, so the first lookup + // still searches. The host binds one disjoint table for the draw. + int lastBinding = gtsContext.GetOrAddRawLastBinding(resourceManager, operation); + Operand last = context.Load(StorageKind.LocalMemory, lastBinding); + Operand lastEntry = context.IMultiply(last, Const(GmemMapEntryWords)); + Operand lastHolds = context.BitwiseAnd( + AtOrBelow(MapWord(lastEntry, 1), MapWord(lastEntry, 0)), + Below(MapWord(lastEntry, 3), MapWord(lastEntry, 2))); + context.Copy(selected, last); + context.BranchIfTrue(lblChosen, lastHolds); + + // A tagged 2 MiB-chunk hint names an entry which holds the whole + // chunk. Collisions and ranges which cover only part of a chunk + // take the search below. + Operand chunk = context.BitwiseOr( + context.ShiftLeft(addrHigh, Const(32 - GmemHintShift)), + context.ShiftRightU32(addrLow, Const(GmemHintShift))); + Operand indexMask = Const((1 << GmemMapLog2) - 1); + Operand hint = MapWord( + context.IMultiply(context.BitwiseAnd(chunk, indexMask), Const(GmemMapEntryWords)), 6); + Operand hintIndex = context.BitwiseAnd(hint, indexMask); + Operand hintFits = context.BitwiseAnd( + context.BitwiseAnd( + context.ICompareNotEqual(hintIndex, Const(0)), + context.ICompareEqual(context.ShiftRightU32(hint, Const(GmemMapLog2)), chunk)), + context.ICompareLessUnsigned(addrHigh, Const(1 << (GmemHintShift - GmemMapLog2)))); + context.Copy(selected, hintIndex); + context.BranchIfTrue(lblFound, hintFits); - for (int step = 1 << (GmemMapLog2 - 1); step != 0; step >>= 1) - { - Operand candidate = context.IAdd(index, Const(step)); - Operand entry = context.IMultiply(candidate, Const(GmemMapEntryWords)); - Operand startLow = MapWord(entry, 0); - Operand startHigh = MapWord(entry, 1); - - Operand atOrBelow = context.BitwiseOr( - context.ICompareLessUnsigned(startHigh, addrHigh), - context.BitwiseAnd(context.ICompareEqual(startHigh, addrHigh), context.ICompareLessOrEqualUnsigned(startLow, addrLow))); - - index = context.ConditionalSelect(atOrBelow, candidate, index); + // The last entry whose start is at or below the address (entry 0 starts at 0, so there always is one) + Operand index = Local(); + Operand step = Local(); + Operand lblSearch = Label(); + context.Copy(index, Const(0)); + context.Copy(step, Const(1 << (GmemMapLog2 - 1))); + context.MarkLabel(lblSearch); + Operand candidate = context.IAdd(index, step); + Operand entry = context.IMultiply(candidate, Const(GmemMapEntryWords)); + context.Copy(index, context.ConditionalSelect(AtOrBelow(MapWord(entry, 1), MapWord(entry, 0)), candidate, index)); + context.Copy(step, context.ShiftRightU32(step, Const(1))); + context.BranchIfTrue(lblSearch, context.ICompareNotEqual(step, Const(0))); + + context.Copy(selected, index); + context.MarkLabel(lblFound); + context.Store(StorageKind.LocalMemory, lastBinding, selected); + Operand found = context.IMultiply(selected, Const(GmemMapEntryWords)); + Operand lblSkip = Label(); + Operand inPool = Below(MapWord(found, 3), MapWord(found, 2)); + + if (vectorCount > 1) + { + // The resolver chose the entry from word zero. Require the + // vector's last byte to remain below its end as well, rather + // than relying on host pool alignment. + Operand lastLow = context.IAdd(addrLow, Const(vectorCount * 4 - 1)); + Operand endCarry = context.ConditionalSelect( + context.ICompareLessUnsigned(lastLow, addrLow), Const(1), Const(0)); + Operand lastHigh = context.IAdd(addrHigh, endCarry); + Operand endLow = MapWord(found, 2); + Operand endHigh = MapWord(found, 3); + inPool = context.BitwiseOr( + context.ICompareLessUnsigned(lastHigh, endHigh), + context.BitwiseAnd( + context.ICompareEqual(lastHigh, endHigh), + context.ICompareLessUnsigned(lastLow, endLow))); } - Operand found = context.IMultiply(index, Const(GmemMapEntryWords)); - Operand endLow = MapWord(found, 2); - Operand endHigh = MapWord(found, 3); - - Operand inPool = context.BitwiseOr( - context.ICompareLessUnsigned(addrHigh, endHigh), - context.BitwiseAnd(context.ICompareEqual(addrHigh, endHigh), context.ICompareLessUnsigned(addrLow, endLow))); - - Operand lblSkip = Label(); context.BranchIfFalse(lblSkip, inPool); + context.MarkLabel(lblChosen); + found = context.IMultiply(selected, Const(GmemMapEntryWords)); + if (writes) { // A page in a pool is never 0 (the guest's first page is not mapped), so 0 marks an empty slot @@ -908,7 +1106,18 @@ Operand MapWord(Operand entry, int word) Operand resultValue = null; - if (operation.Inst == Instruction.Load) + if (operation.Inst == Instruction.Load && vectorCount > 1) + { + resultValue = context.Add( + Instruction.Load, + StorageKind.GlobalMemoryVector, + Local(), + hostLow, + hostHigh, + Const(vectorCount), + Const(resourceManager.GetRawVectorMemory())); + } + else if (operation.Inst == Instruction.Load) { resultValue = context.Load(StorageKind.GlobalMemory, hostLow, hostHigh); @@ -967,11 +1176,23 @@ Operand MapWord(Operand entry, int word) } context.MarkLabel(lblSkip); + context.Store(StorageKind.LocalMemory, lastBinding, Const(0)); bool returnsValue = operation.Dest != null; if (returnsValue) { + // A miss must preserve the scalar semantics: every word reads + // zero, including the followers stored in gmem_vec. + for (int word = 0; word < vectorCount - 1; word++) + { + context.Store( + StorageKind.LocalMemory, + resourceManager.GetRawVectorMemory(), + Const(word), + Const(0)); + } + context.Return(Const(0)); } else @@ -981,12 +1202,15 @@ Operand MapWord(Operand entry, int word) Function function = new( ControlFlowGraph.Create(context.GetOperations()).Blocks, - GetFunctionName(operation, isMultiTarget: true, []) + "Raw", + GetFunctionName(operation, isMultiTarget: true, []) + + (vectorCount > 1 ? $"RawV{vectorCount}" : "Raw"), returnsValue, inArgumentsCount, 0); - functionId = gtsContext.AddFunction(operation, isMultiTarget: true, [], function); + functionId = vectorCount > 1 + ? gtsContext.AddRawVectorFunction(vectorCount, function) + : gtsContext.AddFunction(operation, isMultiTarget: true, [], function); return true; } diff --git a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs index 0a4c0ff4..deb9ce19 100644 --- a/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs +++ b/dm/third_party/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs @@ -159,6 +159,27 @@ public void SetVertexAsComputeLocalMemories(ShaderStage stage, InputTopology inp } } + // nxnative: the pool table entry a raw global memory helper chose last in this invocation (GlobalToStorage.TryGenerateRawStorageOp), one word for each helper, zero at the start + public int AddRawLastEntryMemory() + { + return Properties.AddLocalMemory(new MemoryDefinition($"gmem_last_{Properties.LocalMemories.Count}", AggregateType.U32, 1, zeroInit: true)); + } + + // Words 1..3 returned by one raw vector load. Followers consume them + // immediately, before another memory access can overwrite the array. + private int _rawVectorMemoryId = -1; + + public int GetRawVectorMemory() + { + if (_rawVectorMemoryId < 0) + { + _rawVectorMemoryId = Properties.AddLocalMemory( + new MemoryDefinition("gmem_vec", AggregateType.Array | AggregateType.U32, 3)); + } + + return _rawVectorMemoryId; + } + private int AddMemoryDefinition(string name, AggregateType type, int arrayLength = 1) { return Properties.AddLocalMemory(new MemoryDefinition(name, type, arrayLength)); diff --git a/dm/third_party/ryujinx/src/Spv.Generator/spirv.cs b/dm/third_party/ryujinx/src/Spv.Generator/spirv.cs index ccbe4714..cc80752f 100644 --- a/dm/third_party/ryujinx/src/Spv.Generator/spirv.cs +++ b/dm/third_party/ryujinx/src/Spv.Generator/spirv.cs @@ -161,6 +161,7 @@ public enum ExecutionMode DerivativeGroupLinearNV = 5290, OutputTrianglesNV = 5298, OutputPrimitivesEXT = 5270, + OutputLinesEXT = OutputLinesNV, OutputTrianglesEXT = 5298, PixelInterlockOrderedEXT = 5366, PixelInterlockUnorderedEXT = 5367, @@ -586,6 +587,8 @@ public enum BuiltIn TaskCountNV = 5274, PrimitiveCountNV = 5275, PrimitiveIndicesNV = 5276, + PrimitivePointIndicesEXT = 5294, + PrimitiveLineIndicesEXT = 5295, PrimitiveTriangleIndicesEXT = 5296, ClipDistancePerViewNV = 5277, CullDistancePerViewNV = 5278,