diff --git a/dm/gfx/nvn/dmsh_chain.h b/dm/gfx/nvn/dmsh_chain.h index 8a02bb5206..de102ea380 100644 --- a/dm/gfx/nvn/dmsh_chain.h +++ b/dm/gfx/nvn/dmsh_chain.h @@ -1,3 +1,3 @@ // Generated file (dm/gfx/tools/dmsh_chain.py --header): the B shader library translation chain version this source tree expects (default options), one for every game; // the host loads only /b-/. Run dmsh_chain.py --header after changing the chain's tools and commit this file with the change -#define DMSH_CHAIN "39eb2f781d07" +#define DMSH_CHAIN "6e3c16385059" diff --git a/dm/gfx/tests/ampere_orin/ampere_orin.sass b/dm/gfx/tests/ampere_orin/ampere_orin.sass new file mode 100644 index 0000000000..98a3ca86ac --- /dev/null +++ b/dm/gfx/tests/ampere_orin/ampere_orin.sass @@ -0,0 +1,9623 @@ +// Each op's program as the translator gets it (dm/gfx/tools/ampere_orin/capture.py): the GPU compiler's machine code with its global +// loads and stores made shared memory ones, disassembled by nvdisasm on the device. Thread t's words lie at shared memory word t * W +// (W = inputs + outputs): its inputs, then its outputs. + +// == fadd_rn +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_rn +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == ffma_rn +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fadd_rz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD.RZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_rz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL.RZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == ffma_rz +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.RZ R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fadd_rm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD.RM R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_rm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL.RM R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == ffma_rm +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.RM R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fadd_rp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD.RP R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_rp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL.RP R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == ffma_rp +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.RP R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fadd_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD.FTZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fadd_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD.SAT R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fsub +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD R7, R0, -R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL.FTZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL.SAT R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == ffma_ftz +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.FTZ R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == ffma_sat +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.SAT R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == ffma_ftz_sat +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.FTZ.SAT R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fadd_abs_neg +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD R7, |R0|, -R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_by_imm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FMUL R7, R2, 0.3333333432674407959 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == fmin +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMNMX R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmax +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMNMX R7, R0, R7, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmin_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMNMX.FTZ R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmax_nan +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMNMX.NAN R7, R0, R7, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmin_nan +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMNMX.NAN R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fcopysign +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ LOP3.LUT R7, R7, 0x80000000, R0, 0xb8, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fabs +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FADD R7, |R2|, -RZ ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == fneg +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FADD R7, -R2, -RZ ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == fsetp +// base register R24 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x38 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R24, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R24, R24, 0x40, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R24] ; + /*01e0*/ LDS R7, [R24+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FSETP.NEU.AND P3, PT, R0.reuse, R7.reuse, PT ; + /*0220*/ FSETP.NE.AND P0, PT, R0.reuse, R7.reuse, PT ; + /*0230*/ FSETP.GEU.AND P4, PT, R0.reuse, R7.reuse, PT ; + /*0240*/ FSETP.GTU.AND P5, PT, R0.reuse, R7.reuse, PT ; + /*0250*/ FSETP.GT.AND P1, PT, R0, R7, PT ; + /*0260*/ SEL R9, RZ, 0x1, P3 ; + /*0270*/ FSETP.GE.AND P2, PT, R0.reuse, R7.reuse, PT ; + /*0280*/ SEL R11, RZ, 0x1, !P0 ; + /*0290*/ STS [R24+0x8], R9 ; + /*02a0*/ SEL R3, RZ, 0x1, P4 ; + /*02b0*/ FSETP.NAN.AND P6, PT, R0, R7, PT ; + /*02c0*/ STS [R24+0xc], R11 ; + /*02d0*/ SEL R7, RZ, 0x1, P5 ; + /*02e0*/ SEL R13, RZ, 0x1, !P1 ; + /*02f0*/ STS [R24+0x10], R3 ; + /*0300*/ SEL R15, RZ, 0x1, !P2 ; + /*0310*/ SEL R17, RZ, 0x1, P0 ; + /*0320*/ STS [R24+0x14], R7 ; + /*0330*/ SEL R19, RZ, 0x1, !P3 ; + /*0340*/ SEL R21, RZ, 0x1, P2 ; + /*0350*/ STS [R24+0x18], R13 ; + /*0360*/ SEL R9, RZ, 0x1, !P4 ; + /*0370*/ SEL R11, RZ, 0x1, P6 ; + /*0380*/ STS [R24+0x1c], R15 ; + /*0390*/ SEL R3, RZ, 0x1, P1 ; + /*03a0*/ SEL R23, RZ, 0x1, !P6 ; + /*03b0*/ STS [R24+0x20], R17 ; + /*03c0*/ SEL R7, RZ, 0x1, !P5 ; + /*03d0*/ STS [R24+0x24], R19 ; + /*03e0*/ STS [R24+0x28], R21 ; + /*03f0*/ STS [R24+0x2c], R3 ; + /*0400*/ STS [R24+0x30], R7 ; + /*0410*/ STS [R24+0x34], R9 ; + /*0420*/ STS [R24+0x38], R11 ; + /*0430*/ STS [R24+0x3c], R23 ; + /*0440*/ EXIT ; + /*0450*/ BRA 0x450; + +// == fsetp_ftz +// base register R14 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x10 +// nop 0x0220 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R14, SR_TID.X ; + /*0010*/ IMAD.U32 R14, R14, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x8 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R14] ; + /*01e0*/ LDS R5, [R14+0x4] ; + /*01f0*/ NOP ; + /*0200*/ FSETP.NEU.FTZ.AND P2, PT, R0.reuse, R5.reuse, PT ; + /*0210*/ FSETP.GEU.FTZ.AND P3, PT, R0.reuse, R5.reuse, PT ; + /*0220*/ FSETP.GTU.FTZ.AND P0, PT, R0, R5, PT ; + /*0230*/ FSETP.NE.FTZ.AND P1, PT, R0, R5, PT ; + /*0240*/ NOP ; + /*0250*/ SEL R7, RZ, 0x1, P2 ; + /*0260*/ SEL R9, RZ, 0x1, P3 ; + /*0270*/ SEL R11, RZ, 0x1, !P0 ; + /*0280*/ STS [R14+0x8], R7 ; + /*0290*/ SEL R13, RZ, 0x1, !P1 ; + /*02a0*/ STS [R14+0xc], R9 ; + /*02b0*/ STS [R14+0x10], R11 ; + /*02c0*/ STS [R14+0x14], R13 ; + /*02d0*/ EXIT ; + /*02e0*/ BRA 0x2e0; + +// == fset_bf +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ FSET.BF.LT.AND R9, R0.reuse, R7.reuse, PT ; + /*0210*/ FSET.BF.NEU.AND R7, R0, R7, PT ; + /*0220*/ STS [R10+0x8], R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fset_u32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R7, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ FSETP.GE.AND P0, PT, R0.reuse, R5.reuse, PT ; + /*0200*/ FSETP.NAN.AND P1, PT, R0, R5, PT ; + /*0210*/ NOP ; + /*0220*/ SEL R7, RZ, 0xffffffff, !P0 ; + /*0230*/ SEL R9, RZ, 0xffffffff, !P1 ; + /*0240*/ STS [R10+0x8], R7 ; + /*0250*/ STS [R10+0xc], R9 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; + +// == ftestp +// base register R14 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x4 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x14 +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R14, SR_TID.X ; + /*0010*/ IMAD.U32 R14, R14, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R14] ; + /*01e0*/ NOP ; + /*01f0*/ NOP ; + /*0200*/ LOP3.LUT R0, R2, 0x7fffffff, RZ, 0xc0, !PT ; + /*0210*/ ISETP.GE.U32.AND P4, PT, R0.reuse, 0x7f800000, PT ; + /*0220*/ ISETP.GE.U32.AND P0, PT, R0.reuse, 0x800000, PT ; + /*0230*/ ISETP.GE.U32.AND P1, PT, R0.reuse, 0x800000, !P4 ; + /*0240*/ ISETP.NE.U32.AND P3, PT, R0.reuse, 0x7f800000, PT ; + /*0250*/ ISETP.GT.U32.AND P2, PT, R0.reuse, 0x7f800000, PT ; + /*0260*/ ISETP.EQ.U32.OR P1, PT, R0, RZ, P1 ; + /*0270*/ ISETP.GT.U32.AND P0, PT, R0, RZ, !P0 ; + /*0280*/ SEL R3, RZ, 0x1, P4 ; + /*0290*/ SEL R7, RZ, 0x1, P3 ; + /*02a0*/ SEL R9, RZ, 0x1, P2 ; + /*02b0*/ STS [R14+0x4], R3 ; + /*02c0*/ SEL R11, RZ, 0x1, !P1 ; + /*02d0*/ SEL R13, RZ, 0x1, !P0 ; + /*02e0*/ STS [R14+0x8], R7 ; + /*02f0*/ STS [R14+0xc], R9 ; + /*0300*/ STS [R14+0x10], R11 ; + /*0310*/ STS [R14+0x14], R13 ; + /*0320*/ EXIT ; + /*0330*/ BRA 0x330; + +// == fsel_cmp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0200 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ FSETP.GT.AND P0, PT, R0, RZ, PT ; + /*0200*/ @!P0 LDS R7, [R8+0x8] ; + /*0210*/ NOP ; + /*0220*/ NOP ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == mufu_ex2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FSETP.GEU.AND P0, PT, R0, -126, PT ; + /*0200*/ @!P0 FMUL R0, R0, 0.5 ; + /*0210*/ MUFU.EX2 R7, R0 ; + /*0220*/ @!P0 FMUL R7, R7, R7 ; + /*0230*/ STS [R8+0x4], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == mufu_ex2_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.EX2 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_lg2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FSETP.GEU.AND P0, PT, |R0|, 1.175494350822287508e-38, PT ; + /*0200*/ @!P0 FMUL R0, R0, 16777216 ; + /*0210*/ MUFU.LG2 R7, R0 ; + /*0220*/ @!P0 FADD R7, R7, -24 ; + /*0230*/ STS [R8+0x4], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == mufu_lg2_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.LG2 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_rcp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ HFMA2.MMA R0, -RZ, RZ, 15, 0 ; + /*01f0*/ NOP ; + /*0200*/ FSETP.GEU.AND P1, PT, |R2|.reuse, 1.175494350822287508e-38, PT ; + /*0210*/ FSETP.GT.AND P0, PT, |R2|, 8.50705917302346158658e+37, PT ; + /*0220*/ FSEL R0, R0, 1, !P1 ; + /*0230*/ FSEL R7, R0, 0.25, !P0 ; + /*0240*/ FMUL R0, R2, R7 ; + /*0250*/ MUFU.RCP R0, R0 ; + /*0260*/ FMUL R7, R7, R0 ; + /*0270*/ STS [R8+0x4], R7 ; + /*0280*/ EXIT ; + /*0290*/ BRA 0x290; + +// == mufu_rcp_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.RCP R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_rsqrt +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FSETP.GEU.AND P0, PT, |R0|, 1.175494350822287508e-38, PT ; + /*0200*/ @!P0 FMUL R0, R0, 16777216 ; + /*0210*/ MUFU.RSQ R7, R0 ; + /*0220*/ @!P0 FMUL R7, R7, 4096 ; + /*0230*/ STS [R8+0x4], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == mufu_rsqrt_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.RSQ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_sqrt +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FSETP.GEU.AND P0, PT, |R0|, 1.175494350822287508e-38, PT ; + /*0200*/ @!P0 FMUL R0, R0, 16777216 ; + /*0210*/ MUFU.SQRT R7, R0 ; + /*0220*/ @!P0 FMUL R7, R7, 0.000244140625 ; + /*0230*/ STS [R8+0x4], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == mufu_sqrt_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.SQRT R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_sin +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FMUL.RZ R7, R2, 0.15915493667125701904 ; + /*0200*/ MUFU.SIN R7, R7 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == mufu_cos +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FMUL.RZ R7, R2, 0.15915493667125701904 ; + /*0200*/ MUFU.COS R7, R7 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == mufu_tanh +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.TANH R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_ex2_f16x2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.EX2.F16 R0, R2 ; + /*0200*/ MUFU.EX2.F16 R7, R2.H1 ; + /*0210*/ PRMT R7, R0, 0x5410, R7 ; + /*0220*/ STS [R8+0x4], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == mufu_tanh_f16x2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.TANH.F16 R0, R2 ; + /*0200*/ MUFU.TANH.F16 R7, R2.H1 ; + /*0210*/ PRMT R7, R0, 0x5410, R7 ; + /*0220*/ STS [R8+0x4], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fdiv_rn +// base register R22 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR36, c[0x0][0x118] +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R16, R5, c[0x0][0x160] +// nop 0x02b0 (feeds only the removed addressing): HFMA2.MMA R3, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x02c0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R16, R3, c[0x0][0x168] + /*0000*/ S2R R22, SR_TID.X ; + /*0010*/ IMAD.U32 R22, R22, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R16, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ LEA R16, R3, R16, 0x5 ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R22] ; + /*01e0*/ LDS R5, [R22+0x4] ; + /*01f0*/ BSSY B6, 0x2d0 ; + /*0200*/ MUFU.RCP R0, R5 ; + /*0210*/ FCHK P0, R4, R5 ; + /*0220*/ FFMA R7, -R5, R0, 1 ; + /*0230*/ FFMA R7, R0, R7, R0 ; + /*0240*/ FFMA R0, R4, R7, RZ ; + /*0250*/ FFMA R6, -R5, R0, R4 ; + /*0260*/ FFMA R0, R7, R6, R0 ; + /*0270*/ @!P0 BRA 0x2c0 ; + /*0280*/ MOV R20, 0x0 ; + /*0290*/ MOV R21, 0x0 ; + /*02a0*/ CALL.ABS.NOINC 0x0 ; + /*02b0*/ MOV R0, R4 ; + /*02c0*/ BSYNC B6 ; + /*02d0*/ NOP ; + /*02e0*/ NOP ; + /*02f0*/ STS [R22+0x8], R0 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == fdiv_full +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R6, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FSETP.GT.AND P0, PT, |R6|.reuse, 8.50705917302346158658e+37, PT ; + /*0220*/ FSETP.GEU.AND P1, PT, |R6|, 1.175494350822287508e-38, PT ; + /*0230*/ @P0 FMUL R6, R6, 0.25 ; + /*0240*/ @P0 FMUL R0, R0, 0.25 ; + /*0250*/ @!P1 FMUL R6, R6, 16777216 ; + /*0260*/ @!P1 FMUL R0, R0, 16777216 ; + /*0270*/ MUFU.RCP R7, R6 ; + /*0280*/ FMUL R7, R7, R0 ; + /*0290*/ STS [R8+0x8], R7 ; + /*02a0*/ EXIT ; + /*02b0*/ BRA 0x2b0; + +// == fdiv_approx +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R6, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FSETP.GEU.AND P0, PT, |R6|, 1.175494350822287508e-38, PT ; + /*0220*/ @!P0 FMUL R6, R6, 16777216 ; + /*0230*/ @!P0 FMUL R0, R0, 16777216 ; + /*0240*/ MUFU.RCP R7, R6 ; + /*0250*/ FMUL R7, R7, R0 ; + /*0260*/ STS [R8+0x8], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == frcp_rn +// base register R22 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x4 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR36, c[0x0][0x118] +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R2, R5, c[0x0][0x160] +// nop 0x02a0 (feeds only the removed addressing): HFMA2.MMA R3, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x02b0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R2, R3, c[0x0][0x168] + /*0000*/ S2R R22, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R22, R22, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R2, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ IMAD R2, R3, 0x20, R2 ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R22] ; + /*01e0*/ BSSY B6, 0x2c0 ; + /*01f0*/ IADD3 R0, R4, 0x1800000, RZ ; + /*0200*/ LOP3.LUT R0, R0, 0x7f800000, RZ, 0xc0, !PT ; + /*0210*/ ISETP.GT.U32.AND P0, PT, R0, 0x1ffffff, PT ; + /*0220*/ @P0 BRA 0x270 ; + /*0230*/ MOV R20, 0x0 ; + /*0240*/ MOV R21, 0x0 ; + /*0250*/ CALL.ABS.NOINC 0x0 ; + /*0260*/ BRA 0x2b0 ; + /*0270*/ MUFU.RCP R3, R4 ; + /*0280*/ FFMA R0, R4, R3, -1 ; + /*0290*/ FADD.FTZ R0, -R0, -RZ ; + /*02a0*/ FFMA R4, R3, R0, R3 ; + /*02b0*/ BSYNC B6 ; + /*02c0*/ NOP ; + /*02d0*/ NOP ; + /*02e0*/ STS [R22+0x4], R4 ; + /*02f0*/ EXIT ; + /*0300*/ BRA 0x300; + +// == fsqrt_rn +// base register R22 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x4 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR36, c[0x0][0x118] +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R2, R5, c[0x0][0x160] +// nop 0x02a0 (feeds only the removed addressing): HFMA2.MMA R3, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x02b0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R2, R3, c[0x0][0x168] + /*0000*/ S2R R22, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R22, R22, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R2, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ LEA R2, R3, R2, 0x5 ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R22] ; + /*01e0*/ BSSY B6, 0x2c0 ; + /*01f0*/ IADD3 R0, R4, -0xd000000, RZ ; + /*0200*/ MUFU.RSQ R3, R4 ; + /*0210*/ ISETP.GT.U32.AND P0, PT, R0, 0x727fffff, PT ; + /*0220*/ @!P0 BRA 0x270 ; + /*0230*/ MOV R20, 0x0 ; + /*0240*/ MOV R21, 0x0 ; + /*0250*/ CALL.ABS.NOINC 0x0 ; + /*0260*/ BRA 0x2b0 ; + /*0270*/ FMUL.FTZ R5, R4, R3 ; + /*0280*/ FMUL.FTZ R3, R3, 0.5 ; + /*0290*/ FFMA R4, -R5, R5, R4 ; + /*02a0*/ FFMA R4, R4, R3, R5 ; + /*02b0*/ BSYNC B6 ; + /*02c0*/ NOP ; + /*02d0*/ NOP ; + /*02e0*/ STS [R22+0x4], R4 ; + /*02f0*/ EXIT ; + /*0300*/ BRA 0x300; + +// == f2i_s32_rni +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == frnd_rni +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FRND R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_s32_rzi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.TRUNC.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == frnd_rzi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FRND.TRUNC R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_s32_rmi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.FLOOR.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == frnd_rmi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FRND.FLOOR R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_s32_rpi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.CEIL.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == frnd_rpi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FRND.CEIL R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_u32_rzi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.U32.TRUNC.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_s32_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.FTZ.TRUNC.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_s16_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.S16.TRUNC.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_u8_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2IP.U8.F32.NTZ R0, RZ, R2, RZ ; + /*0200*/ LOP3.LUT R7, R0, 0xff, RZ, 0xc0, !PT ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == f2i_s64 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R6, R3, R6, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R6, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): HFMA2.MMA R7, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R6, R7, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R6, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ NOP ; + /*0200*/ F2I.S64.TRUNC R4, R2 ; + /*0210*/ STS [R8+0x4], R4 ; + /*0220*/ STS [R8+0x8], R5 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == i2f_s32_rn +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2FP.F32.S32 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_s32_rz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2FP.F32.S32.RZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_s32_rm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2F.RM R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_s32_rp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2F.RP R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_u32_rn +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2FP.F32.U32 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_s16 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2F.S16 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_s64 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R6, R3, R6, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R6, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R7, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R6, R7, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R6, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R8] ; + /*01e0*/ LDS R5, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ I2F.S64 R5, R4 ; + /*0220*/ STS [R8+0x8], R5 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == f2f_f16_rn +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2F.F16.F32 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2f_f16_rz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2F.F16.F32.RZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2f_f32_from_f16 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R10] ; + /*01e0*/ NOP ; + /*01f0*/ NOP ; + /*0200*/ HADD2.F32 R7, -RZ, R2.reuse.H0_H0 ; + /*0210*/ HADD2.F32 R9, -RZ, R2.H1_H1 ; + /*0220*/ STS [R10+0x4], R7 ; + /*0230*/ STS [R10+0x8], R9 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == f2fp_pack_f16x2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ F2FP.PACK_AB R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == f2fp_pack_f16x2_relu +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ F2FP.RELU.PACK_AB R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == f2fp_pack_bf16x2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ F2FP.BF16.PACK_AB R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == f2f_tf32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ HFMA2.MMA R7, -RZ, RZ, 0, 0.00048828125 ; + /*01f0*/ NOP ; + /*0200*/ FSETP.GEU.AND P0, PT, |R0|, +INF , PT ; + /*0210*/ @!P0 IMAD.IADD.U32 R0, R0, 0x1, R7 ; + /*0220*/ LOP3.LUT R7, R0, 0xffffe000, RZ, 0xc0, !PT ; + /*0230*/ STS [R8+0x4], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == i2i_s8_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2I.S8.S32.SAT R0, R2 ; + /*0200*/ PRMT R7, R0, 0x8880, RZ ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == i2i_u16_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2I.U16.S32.SAT R0, R2 ; + /*0200*/ LOP3.LUT R7, R0, 0xffff, RZ, 0xc0, !PT ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == sext_s8 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ PRMT R7, R2, 0x8880, RZ ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == hadd2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HADD2 R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hadd2_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HADD2.FTZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hadd2_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HADD2.SAT R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hsub2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HADD2 R7, R0, -R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hmul2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMUL2 R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hmul2_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMUL2.FTZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hfma2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ HFMA2.MMA R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == hfma2_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ HFMA2.MMA.FTZ R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == hfma2_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ HFMA2.MMA.SAT R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == hfma2_relu +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ HFMA2.MMA.RELU R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == hmin2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMNMX2 R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hmax2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMNMX2 R7, R0, R7, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hmin2_nan +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMNMX2.NAN R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hneg2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ HADD2 R7, -R2, -RZ.H0_H0 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == habs2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ HADD2 R7, |R2|, -RZ.H0_H0 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == hadd2_imm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ HADD2 R7, R2, 1, -1 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == hadd_f16 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R7, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ HADD2 R3, R0.reuse.H0_H0, R5.H0_H0 ; + /*0200*/ HMUL2 R6, R0.H1_H1, R5.H1_H1 ; + /*0210*/ NOP ; + /*0220*/ PRMT R0, R3, 0x7610, R0 ; + /*0230*/ LOP3.LUT R9, R6, 0xffff, RZ, 0xc0, !PT ; + /*0240*/ LOP3.LUT R7, R0, 0xffff, RZ, 0xc0, !PT ; + /*0250*/ STS [R10+0x8], R7 ; + /*0260*/ STS [R10+0xc], R9 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == hsetp2 +// base register R20 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x20 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R20, SR_TID.X ; + /*0010*/ IMAD.U32 R20, R20, 0x28, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R20] ; + /*01e0*/ LDS R7, [R20+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HSETP2.LT.AND P1, P2, R0.reuse, R7.reuse, PT ; + /*0220*/ HSETP2.EQ.AND P3, P0, R0, R7, PT ; + /*0230*/ SEL R9, RZ, 0x1, !P1 ; + /*0240*/ SEL R11, RZ, 0x1, !P2 ; + /*0250*/ HSETP2.GTU.AND P1, P2, R0.reuse, R7.reuse, PT ; + /*0260*/ STS [R20+0x8], R9 ; + /*0270*/ SEL R13, RZ, 0x1, !P3 ; + /*0280*/ HSETP2.NAN.AND P3, P4, R0, R7, PT ; + /*0290*/ STS [R20+0xc], R11 ; + /*02a0*/ SEL R3, RZ, 0x1, !P0 ; + /*02b0*/ SEL R7, RZ, 0x1, !P1 ; + /*02c0*/ STS [R20+0x10], R13 ; + /*02d0*/ SEL R15, RZ, 0x1, !P2 ; + /*02e0*/ SEL R17, RZ, 0x1, !P3 ; + /*02f0*/ STS [R20+0x14], R3 ; + /*0300*/ SEL R19, RZ, 0x1, !P4 ; + /*0310*/ STS [R20+0x18], R7 ; + /*0320*/ STS [R20+0x1c], R15 ; + /*0330*/ STS [R20+0x20], R17 ; + /*0340*/ STS [R20+0x24], R19 ; + /*0350*/ EXIT ; + /*0360*/ BRA 0x360; + +// == hset2 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ HSET2.BF.LT.AND R9, R0.reuse, R7.reuse, PT ; + /*0210*/ HSET2.GEU.AND R7, R0, R7, PT ; + /*0220*/ STS [R10+0x8], R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == bfma2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ HFMA2.BF16_V2 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == bmin2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMNMX2.BF16_V2 R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == bmax2_nan +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMNMX2.BF16_V2.NAN R7, R0, R7, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == iadd +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IADD3 R7, R0, R7, RZ ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == iadd3 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IADD3 R7, -R6, R0, R7 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == iadd_sat +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R9, RZ, RZ, 0x4 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x8 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ IADD3 R6, R0, R5, RZ ; + /*0210*/ PLOP3.LUT P0, PT, R0.SIGN, R5.SIGN, R6.SIGN, 0x2, 0x0 ; + /*0220*/ PLOP3.LUT P1, PT, R0.SIGN, R5.SIGN, R6.SIGN, 0x40, 0x0 ; + /*0230*/ NOP ; + /*0240*/ SEL R7, R6, 0x7fffffff, !P0 ; + /*0250*/ SEL R7, R7, 0x80000000, !P1 ; + /*0260*/ STS [R10+0x8], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == isub_sat +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R9, RZ, RZ, 0x4 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x8 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ IADD3 R6, R0, -R5, RZ ; + /*0210*/ PLOP3.LUT P0, PT, R0.SIGN, R5.SIGN, R6.SIGN, 0x8, 0x0 ; + /*0220*/ PLOP3.LUT P1, PT, R0.SIGN, R5.SIGN, R6.SIGN, 0x10, 0x0 ; + /*0230*/ NOP ; + /*0240*/ SEL R7, R6, 0x7fffffff, !P0 ; + /*0250*/ SEL R7, R7, 0x80000000, !P1 ; + /*0260*/ STS [R10+0x8], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == iadd64 +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R11, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R11, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 9.5367431640625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R12] ; + /*01e0*/ LDS R6, [R12+0x4] ; + /*01f0*/ LDS R5, [R12+0x8] ; + /*0200*/ LDS R7, [R12+0xc] ; + /*0210*/ NOP ; + /*0220*/ IADD3 R9, P0, R0, R5, RZ ; + /*0230*/ NOP ; + /*0240*/ IADD3.X R7, R6, R7, RZ, P0, !PT ; + /*0250*/ STS [R12+0x10], R9 ; + /*0260*/ STS [R12+0x14], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == isub64 +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R11, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R11, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 9.5367431640625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R12] ; + /*01e0*/ LDS R6, [R12+0x4] ; + /*01f0*/ LDS R5, [R12+0x8] ; + /*0200*/ LDS R7, [R12+0xc] ; + /*0210*/ NOP ; + /*0220*/ IADD3 R9, P0, R0, -R5, RZ ; + /*0230*/ NOP ; + /*0240*/ IADD3.X R7, R6, ~R7, RZ, P0, !PT ; + /*0250*/ STS [R12+0x10], R9 ; + /*0260*/ STS [R12+0x14], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == imul_lo +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IMAD R7, R0, R7, RZ ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imul_hi_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IMAD.HI.U32 R7, R0, R7, RZ ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imul_hi_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IMAD.HI R7, R0, R7, RZ ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imul_wide_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R7, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R6, R3, R6, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R6, R7, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R6, R7, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R6, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R8] ; + /*01e0*/ LDS R5, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ IMAD.WIDE R4, R4, R5, RZ ; + /*0210*/ STS [R8+0x8], R4 ; + /*0220*/ STS [R8+0xc], R5 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imul_wide_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R7, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R6, R3, R6, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R6, R7, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R6, R7, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R6, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R8] ; + /*01e0*/ LDS R5, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ IMAD.WIDE.U32 R4, R4, R5, RZ ; + /*0210*/ STS [R8+0x8], R4 ; + /*0220*/ STS [R8+0xc], R5 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imad_lo +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IMAD R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == imad_hi_s32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0200 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R9, [R10] ; + /*01e0*/ LDS R0, [R10+0x4] ; + /*01f0*/ LDS R7, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ MOV R6, RZ ; + /*0220*/ NOP ; + /*0230*/ IMAD.HI R9, R9, R0, R6 ; + /*0240*/ STS [R10+0xc], R9 ; + /*0250*/ EXIT ; + /*0260*/ BRA 0x260; + +// == imul64 +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R11, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0230 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R11, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 9.5367431640625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R12] ; + /*01e0*/ LDS R5, [R12+0x4] ; + /*01f0*/ LDS R6, [R12+0x8] ; + /*0200*/ LDS R9, [R12+0xc] ; + /*0210*/ NOP ; + /*0220*/ IMAD R5, R5, R6, RZ ; + /*0230*/ IMAD.WIDE.U32 R6, R0, R6, RZ ; + /*0240*/ IMAD R9, R0, R9, R5 ; + /*0250*/ NOP ; + /*0260*/ IADD3 R7, R7, R9, RZ ; + /*0270*/ STS [R12+0x10], R6 ; + /*0280*/ STS [R12+0x14], R7 ; + /*0290*/ EXIT ; + /*02a0*/ BRA 0x2a0; + +// == imul24 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0200 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ SGXT R0, R0, 0x18 ; + /*0210*/ SGXT R7, R5, 0x18 ; + /*0220*/ NOP ; + /*0230*/ IMAD R7, R0, R7, RZ ; + /*0240*/ STS [R10+0x8], R7 ; + /*0250*/ EXIT ; + /*0260*/ BRA 0x260; + +// == imin_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IMNMX R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imax_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IMNMX.U32 R7, R0, R7, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imin_s64 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x10 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0200 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R6, [R10+0x4] ; + /*01f0*/ LDS R7, [R10+0x8] ; + /*0200*/ LDS R9, [R10+0xc] ; + /*0210*/ NOP ; + /*0220*/ NOP ; + /*0230*/ ISETP.LT.U32.AND P0, PT, R0, R7, PT ; + /*0240*/ ISETP.LT.AND.EX P0, PT, R6, R9, PT, P0 ; + /*0250*/ SEL R7, R0, R7, P0 ; + /*0260*/ SEL R9, R6, R9, P0 ; + /*0270*/ STS [R10+0x10], R7 ; + /*0280*/ STS [R10+0x14], R9 ; + /*0290*/ EXIT ; + /*02a0*/ BRA 0x2a0; + +// == iabs +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ IABS R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == ineg +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ IADD3 R7, -R2, RZ, RZ ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == isad +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ VABSDIFF.U32 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == idiv_u32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R0, R3, R0, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R3, RZ, RZ, 0x4 +// nop 0x02a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R3, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R6, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ I2F.U32.RP R8, R7 ; + /*0210*/ IMAD.MOV R9, RZ, RZ, -R7 ; + /*0220*/ ISETP.NE.U32.AND P2, PT, R7, RZ, PT ; + /*0230*/ MUFU.RCP R8, R8 ; + /*0240*/ IADD3 R4, R8, 0xffffffe, RZ ; + /*0250*/ F2I.FTZ.U32.TRUNC.NTZ R5, R4 ; + /*0260*/ HFMA2.MMA R4, -RZ, RZ, 0, 0 ; + /*0270*/ IMAD R9, R9, R5, RZ ; + /*0280*/ IMAD.HI.U32 R5, R5, R9, R4 ; + /*0290*/ IMAD.HI.U32 R5, R5, R6, RZ ; + /*02a0*/ IMAD.MOV R2, RZ, RZ, -R5 ; + /*02b0*/ IMAD R6, R7, R2, R6 ; + /*02c0*/ NOP ; + /*02d0*/ ISETP.GE.U32.AND P0, PT, R6, R7, PT ; + /*02e0*/ @P0 IADD3 R6, -R7, R6, RZ ; + /*02f0*/ @P0 IADD3 R5, R5, 0x1, RZ ; + /*0300*/ ISETP.GE.U32.AND P1, PT, R6, R7, PT ; + /*0310*/ @P1 IADD3 R5, R5, 0x1, RZ ; + /*0320*/ @!P2 LOP3.LUT R5, RZ, R7, RZ, 0x33, !PT ; + /*0330*/ STS [R10+0x8], R5 ; + /*0340*/ EXIT ; + /*0350*/ BRA 0x350; + +// == irem_u32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R0, R3, 0x20, R0 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R3, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R3, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R6, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ I2F.U32.RP R8, R7 ; + /*0220*/ IADD3 R9, RZ, -R7, RZ ; + /*0230*/ ISETP.NE.U32.AND P1, PT, R7, RZ, PT ; + /*0240*/ MUFU.RCP R8, R8 ; + /*0250*/ IADD3 R4, R8, 0xffffffe, RZ ; + /*0260*/ F2I.FTZ.U32.TRUNC.NTZ R5, R4 ; + /*0270*/ IMAD.MOV.U32 R4, RZ, RZ, RZ ; + /*0280*/ IMAD R9, R9, R5, RZ ; + /*0290*/ IMAD.HI.U32 R5, R5, R9, R4 ; + /*02a0*/ IMAD.HI.U32 R5, R5, R6, RZ ; + /*02b0*/ IADD3 R5, -R5, RZ, RZ ; + /*02c0*/ IMAD R6, R7, R5, R6 ; + /*02d0*/ ISETP.GE.U32.AND P0, PT, R6, R7, PT ; + /*02e0*/ @P0 IMAD.IADD R6, R6, 0x1, -R7 ; + /*02f0*/ ISETP.GE.U32.AND P0, PT, R6, R7, PT ; + /*0300*/ @P0 IADD3 R6, -R7, R6, RZ ; + /*0310*/ @!P1 LOP3.LUT R6, RZ, R7, RZ, 0x33, !PT ; + /*0320*/ STS [R10+0x8], R6 ; + /*0330*/ EXIT ; + /*0340*/ BRA 0x340; + +// == idiv_s32 +// base register R13 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R0, R3, 0x20, R0 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x02e0 (feeds only the removed addressing): IMAD.MOV.U32 R3, RZ, RZ, 0x4 +// nop 0x0330 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R3, c[0x0][0x168] + /*0000*/ S2R R13, SR_TID.X ; + /*0010*/ IMAD.U32 R13, R13, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R6, [R13] ; + /*01e0*/ LDS R7, [R13+0x4] ; + /*01f0*/ IABS R12, R6 ; + /*0200*/ IABS R9, R7.reuse ; + /*0210*/ IABS R3, R7 ; + /*0220*/ I2F.RP R8, R9 ; + /*0230*/ MOV R2, R12 ; + /*0240*/ IMAD.MOV R3, RZ, RZ, -R3 ; + /*0250*/ LOP3.LUT R6, R6, R7, RZ, 0x3c, !PT ; + /*0260*/ ISETP.GE.AND P1, PT, R6, RZ, PT ; + /*0270*/ MUFU.RCP R8, R8 ; + /*0280*/ IADD3 R4, R8, 0xffffffe, RZ ; + /*0290*/ F2I.FTZ.U32.TRUNC.NTZ R5, R4 ; + /*02a0*/ IMAD.MOV.U32 R4, RZ, RZ, RZ ; + /*02b0*/ IMAD.MOV R10, RZ, RZ, -R5 ; + /*02c0*/ IMAD R11, R10, R9, RZ ; + /*02d0*/ IMAD.HI.U32 R5, R5, R11, R4 ; + /*02e0*/ IMAD.HI.U32 R5, R5, R2, RZ ; + /*02f0*/ IMAD R2, R5, R3, R2 ; + /*0300*/ NOP ; + /*0310*/ ISETP.GT.U32.AND P2, PT, R9, R2, PT ; + /*0320*/ @!P2 IADD3 R2, R2, -R9.reuse, RZ ; + /*0330*/ @!P2 IADD3 R5, R5, 0x1, RZ ; + /*0340*/ ISETP.GE.U32.AND P0, PT, R2, R9, PT ; + /*0350*/ NOP ; + /*0360*/ ISETP.NE.AND P2, PT, R7, RZ, PT ; + /*0370*/ @P0 IADD3 R5, R5, 0x1, RZ ; + /*0380*/ @!P1 IADD3 R5, -R5, RZ, RZ ; + /*0390*/ @!P2 LOP3.LUT R5, RZ, R7, RZ, 0x33, !PT ; + /*03a0*/ STS [R13+0x8], R5 ; + /*03b0*/ EXIT ; + /*03c0*/ BRA 0x3c0; + +// == irem_s32 +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R0, R3, 0x20, R0 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x02d0 (feeds only the removed addressing): MOV R3, 0x4 +// nop 0x02f0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R3, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R11, [R12] ; + /*01e0*/ LDS R6, [R12+0x4] ; + /*01f0*/ IABS R10, R11 ; + /*0200*/ ISETP.GE.AND P2, PT, R11, RZ, PT ; + /*0210*/ IABS R8, R6.reuse ; + /*0220*/ IMAD.MOV.U32 R2, RZ, RZ, R10 ; + /*0230*/ IABS R3, R6 ; + /*0240*/ I2F.RP R7, R8 ; + /*0250*/ IMAD.MOV R3, RZ, RZ, -R3 ; + /*0260*/ MUFU.RCP R7, R7 ; + /*0270*/ IADD3 R4, R7, 0xffffffe, RZ ; + /*0280*/ F2I.FTZ.U32.TRUNC.NTZ R5, R4 ; + /*0290*/ HFMA2.MMA R4, -RZ, RZ, 0, 0 ; + /*02a0*/ IADD3 R9, RZ, -R5, RZ ; + /*02b0*/ IMAD R9, R9, R8, RZ ; + /*02c0*/ IMAD.HI.U32 R5, R5, R9, R4 ; + /*02d0*/ IMAD.HI.U32 R5, R5, R2, RZ ; + /*02e0*/ IMAD R5, R5, R3, R2 ; + /*02f0*/ NOP ; + /*0300*/ ISETP.GT.U32.AND P0, PT, R8, R5, PT ; + /*0310*/ NOP ; + /*0320*/ @!P0 IADD3 R5, R5, -R8, RZ ; + /*0330*/ ISETP.NE.AND P0, PT, R6, RZ, PT ; + /*0340*/ ISETP.GT.U32.AND P1, PT, R8, R5, PT ; + /*0350*/ @!P1 IMAD.IADD R5, R5, 0x1, -R8 ; + /*0360*/ @!P2 IMAD.MOV R5, RZ, RZ, -R5 ; + /*0370*/ @!P0 LOP3.LUT R5, RZ, R6, RZ, 0x33, !PT ; + /*0380*/ STS [R12+0x8], R5 ; + /*0390*/ EXIT ; + /*03a0*/ BRA 0x3a0; + +// == idiv_u32_by_7 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R0, R3, R0, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R9, c[0x0][0x160] +// nop 0x0250 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R10] ; + /*01e0*/ I2F.U32.RP R6, 0x7 ; + /*01f0*/ MUFU.RCP R6, R6 ; + /*0200*/ IADD3 R4, R6, 0xffffffe, RZ ; + /*0210*/ F2I.FTZ.U32.TRUNC.NTZ R5, R4 ; + /*0220*/ HFMA2.MMA R4, -RZ, RZ, 0, 0 ; + /*0230*/ IMAD R7, R5, -0x7, RZ ; + /*0240*/ IMAD.HI.U32 R5, R5, R7, R4 ; + /*0250*/ IMAD.HI.U32 R5, R5, R2, RZ ; + /*0260*/ IMAD R7, R5, -0x7, R2 ; + /*0270*/ NOP ; + /*0280*/ ISETP.GE.U32.AND P0, PT, R7, 0x7, PT ; + /*0290*/ @P0 IADD3 R7, R7, -0x7, RZ ; + /*02a0*/ @P0 IADD3 R5, R5, 0x1, RZ ; + /*02b0*/ ISETP.GE.U32.AND P1, PT, R7, 0x7, PT ; + /*02c0*/ @P1 IADD3 R5, R5, 0x1, RZ ; + /*02d0*/ STS [R10+0x4], R5 ; + /*02e0*/ EXIT ; + /*02f0*/ BRA 0x2f0; + +// == isetp +// base register R16 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x18 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R16, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R16, R16, 0x20, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R16] ; + /*01e0*/ LDS R7, [R16+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ ISETP.GE.AND P1, PT, R0.reuse, R7.reuse, PT ; + /*0220*/ ISETP.GE.U32.AND P0, PT, R0.reuse, R7.reuse, PT ; + /*0230*/ SEL R9, RZ, 0x1, P1 ; + /*0240*/ ISETP.GE.AND P1, PT, R0.reuse, R7.reuse, PT ; + /*0250*/ ISETP.GT.U32.AND P2, PT, R0.reuse, R7.reuse, PT ; + /*0260*/ STS [R16+0x8], R9 ; + /*0270*/ ISETP.NE.AND P4, PT, R0, R7, PT ; + /*0280*/ ISETP.NE.U32.AND P3, PT, R0, R7, PT ; + /*0290*/ SEL R3, RZ, 0x1, P0 ; + /*02a0*/ SEL R7, RZ, 0x1, !P1 ; + /*02b0*/ SEL R11, RZ, 0x1, !P2 ; + /*02c0*/ STS [R16+0xc], R3 ; + /*02d0*/ SEL R13, RZ, 0x1, P4 ; + /*02e0*/ SEL R15, RZ, 0x1, !P3 ; + /*02f0*/ STS [R16+0x10], R7 ; + /*0300*/ STS [R16+0x14], R11 ; + /*0310*/ STS [R16+0x18], R13 ; + /*0320*/ STS [R16+0x1c], R15 ; + /*0330*/ EXIT ; + /*0340*/ BRA 0x340; + +// == isetp_and +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0200 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x8 +// nop 0x0230 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x10 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ LDS R7, [R10+0xc] ; + /*0210*/ ISETP.GE.AND P0, PT, R6, R7, PT ; + /*0220*/ NOP ; + /*0230*/ ISETP.LT.AND P1, PT, R0.reuse, R5.reuse, !P0 ; + /*0240*/ ISETP.GE.U32.OR P0, PT, R0, R5, !P0 ; + /*0250*/ NOP ; + /*0260*/ SEL R7, RZ, 0x1, !P1 ; + /*0270*/ SEL R9, RZ, 0x1, !P0 ; + /*0280*/ STS [R10+0x10], R7 ; + /*0290*/ STS [R10+0x14], R9 ; + /*02a0*/ EXIT ; + /*02b0*/ BRA 0x2b0; + +// == isetp64 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0200 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x8 +// nop 0x0230 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x10 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R7, [R10+0x8] ; + /*0200*/ LDS R6, [R10+0xc] ; + /*0210*/ ISETP.GE.U32.AND P0, PT, R0, R7, PT ; + /*0220*/ NOP ; + /*0230*/ ISETP.GE.AND.EX P1, PT, R5.reuse, R6.reuse, PT, P0 ; + /*0240*/ ISETP.GE.U32.AND.EX P0, PT, R5, R6, PT, P0 ; + /*0250*/ NOP ; + /*0260*/ SEL R7, RZ, 0x1, P1 ; + /*0270*/ SEL R9, RZ, 0x1, P0 ; + /*0280*/ STS [R10+0x10], R7 ; + /*0290*/ STS [R10+0x14], R9 ; + /*02a0*/ EXIT ; + /*02b0*/ BRA 0x2b0; + +// == plop3 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x4 +// nop 0x0220 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0xc ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R6, [R8] ; + /*01e0*/ LDS R0, [R8+0x4] ; + /*01f0*/ LDS R5, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ ISETP.NE.U32.AND P0, PT, R0, RZ, PT ; + /*0220*/ ISETP.NE.U32.XOR P0, PT, R6, RZ, P0 ; + /*0230*/ ISETP.NE.U32.OR P0, PT, R5, RZ, P0 ; + /*0240*/ NOP ; + /*0250*/ SEL R7, RZ, 0x1, P0 ; + /*0260*/ STS [R8+0xc], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == shl +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHF.L.U32 R7, R0, R7, RZ ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == shr_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHF.R.U32.HI R7, RZ, R7, R0 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == shr_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHF.R.S32.HI R7, RZ, R7, R0 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == shl_imm +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R10] ; + /*01e0*/ NOP ; + /*01f0*/ NOP ; + /*0200*/ SHF.L.U32 R7, R2, 0x7, RZ ; + /*0210*/ SHF.R.S32.HI R9, RZ, 0xd, R2 ; + /*0220*/ STS [R10+0x4], R7 ; + /*0230*/ STS [R10+0x8], R9 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shf_l_wrap +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.L.W.U32.HI R7, R0, R6, R7 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shf_l_clamp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.L.U32.HI R7, R0, R6, R7 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shf_r_wrap +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.R.W.U32 R7, R0, R6, R7 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shf_r_clamp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.R.U32 R7, R0, R6, R7 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shl64 +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R12] ; + /*01e0*/ LDS R7, [R12+0x4] ; + /*01f0*/ LDS R9, [R12+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.L.U32 R11, R0.reuse, R9.reuse, RZ ; + /*0230*/ SHF.L.U64.HI R7, R0, R9, R7 ; + /*0240*/ STS [R12+0xc], R11 ; + /*0250*/ STS [R12+0x10], R7 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; + +// == shr64_u +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.R.U64 R9, R0, R6.reuse, R7.reuse ; + /*0230*/ SHF.R.U32.HI R7, RZ, R6, R7 ; + /*0240*/ STS [R10+0xc], R9 ; + /*0250*/ STS [R10+0x10], R7 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; + +// == shr64_s +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.R.S64 R9, R0, R6.reuse, R7.reuse ; + /*0230*/ SHF.R.S32.HI R7, RZ, R6, R7 ; + /*0240*/ STS [R10+0xc], R9 ; + /*0250*/ STS [R10+0x10], R7 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; + +// == lop3 +// base register R24 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0xc +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x28 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R24, SR_TID.X ; + /*0010*/ IMAD.U32 R24, R24, 0x34, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R24] ; + /*01e0*/ LDS R7, [R24+0x4] ; + /*01f0*/ LDS R6, [R24+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ LOP3.LUT R9, R0.reuse, R7.reuse, R6.reuse, 0x96, !PT ; + /*0230*/ LOP3.LUT R11, R0.reuse, R7.reuse, R6.reuse, 0xe8, !PT ; + /*0240*/ LOP3.LUT R13, R0.reuse, R7.reuse, R6.reuse, 0xca, !PT ; + /*0250*/ STS [R24+0xc], R9 ; + /*0260*/ LOP3.LUT R15, R0.reuse, R7.reuse, R6.reuse, 0x1, !PT ; + /*0270*/ LOP3.LUT R17, R0.reuse, R7.reuse, R6.reuse, 0x80, !PT ; + /*0280*/ STS [R24+0x10], R11 ; + /*0290*/ LOP3.LUT R19, R0, R7, R6, 0x6a, !PT ; + /*02a0*/ LOP3.LUT R21, R0.reuse, R7.reuse, R6.reuse, 0xf8, !PT ; + /*02b0*/ STS [R24+0x14], R13 ; + /*02c0*/ LOP3.LUT R23, R0.reuse, R7.reuse, R6.reuse, 0x1e, !PT ; + /*02d0*/ LOP3.LUT R3, R0.reuse, R7.reuse, R6.reuse, 0xb4, !PT ; + /*02e0*/ STS [R24+0x18], R15 ; + /*02f0*/ LOP3.LUT R7, R0, R7, R6, 0x3c, !PT ; + /*0300*/ STS [R24+0x1c], R17 ; + /*0310*/ STS [R24+0x20], R19 ; + /*0320*/ STS [R24+0x24], R21 ; + /*0330*/ STS [R24+0x28], R23 ; + /*0340*/ STS [R24+0x2c], R3 ; + /*0350*/ STS [R24+0x30], R7 ; + /*0360*/ EXIT ; + /*0370*/ BRA 0x370; + +// == logic +// base register R14 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x10 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R14, SR_TID.X ; + /*0010*/ IMAD.U32 R14, R14, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R14] ; + /*01e0*/ LDS R7, [R14+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ LOP3.LUT R9, R0.reuse, R7.reuse, RZ, 0xc0, !PT ; + /*0220*/ LOP3.LUT R11, R0.reuse, R7.reuse, RZ, 0xfc, !PT ; + /*0230*/ LOP3.LUT R13, R0.reuse, R7.reuse, RZ, 0x3c, !PT ; + /*0240*/ STS [R14+0x8], R9 ; + /*0250*/ LOP3.LUT R7, R0, R7, RZ, 0xc, !PT ; + /*0260*/ STS [R14+0xc], R11 ; + /*0270*/ STS [R14+0x10], R13 ; + /*0280*/ STS [R14+0x14], R7 ; + /*0290*/ EXIT ; + /*02a0*/ BRA 0x2a0; + +// == prmt +// base register R20 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0xc +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x1c +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R20, SR_TID.X ; + /*0010*/ IMAD.U32 R20, R20, 0x28, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R20] ; + /*01e0*/ LDS R7, [R20+0x4] ; + /*01f0*/ LDS R6, [R20+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ PRMT R9, R0.reuse, R6.reuse, R7.reuse ; + /*0230*/ PRMT.F4E R11, R0.reuse, R6.reuse, R7.reuse ; + /*0240*/ PRMT.B4E R13, R0.reuse, R6.reuse, R7.reuse ; + /*0250*/ STS [R20+0xc], R9 ; + /*0260*/ PRMT.RC8 R15, R0.reuse, R6.reuse, R7.reuse ; + /*0270*/ PRMT.ECL R17, R0.reuse, R6.reuse, R7.reuse ; + /*0280*/ STS [R20+0x10], R11 ; + /*0290*/ PRMT.ECR R19, R0, R6, R7 ; + /*02a0*/ PRMT.RC16 R7, R0, R6, R7 ; + /*02b0*/ STS [R20+0x14], R13 ; + /*02c0*/ STS [R20+0x18], R15 ; + /*02d0*/ STS [R20+0x1c], R17 ; + /*02e0*/ STS [R20+0x20], R19 ; + /*02f0*/ STS [R20+0x24], R7 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == prmt_imm +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0xc +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R12] ; + /*01e0*/ LDS R7, [R12+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ PRMT R9, R0.reuse, 0x3210, R7.reuse ; + /*0220*/ PRMT R11, R0.reuse, 0x5140, R7.reuse ; + /*0230*/ PRMT R7, R0, 0x8880, R7 ; + /*0240*/ STS [R12+0x8], R9 ; + /*0250*/ STS [R12+0xc], R11 ; + /*0260*/ STS [R12+0x10], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == bfe_u32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x4 +// nop 0x0220 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0xc ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ PRMT R5, R6, 0x7604, R5 ; + /*0220*/ PRMT R9, RZ, 0x4, R5.reuse ; + /*0230*/ PRMT R6, RZ, 0x5, R5 ; + /*0240*/ NOP ; + /*0250*/ SHF.R.U32.HI R9, RZ, R9, R0 ; + /*0260*/ SGXT.U32 R7, R9, R6 ; + /*0270*/ STS [R10+0xc], R7 ; + /*0280*/ EXIT ; + /*0290*/ BRA 0x290; + +// == bfe_s32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x4 +// nop 0x0220 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0xc ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ PRMT R5, R6, 0x7604, R5 ; + /*0220*/ PRMT R9, RZ, 0x4, R5.reuse ; + /*0230*/ PRMT R6, RZ, 0x5, R5 ; + /*0240*/ NOP ; + /*0250*/ SHF.R.S32.HI R9, RZ, R9, R0 ; + /*0260*/ SGXT R7, R9, R6 ; + /*0270*/ STS [R10+0xc], R7 ; + /*0280*/ EXIT ; + /*0290*/ BRA 0x290; + +// == bfi +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0220 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x4 +// nop 0x0240 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x10 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R6, [R10+0x4] ; + /*01f0*/ LDS R5, [R10+0x8] ; + /*0200*/ LDS R8, [R10+0xc] ; + /*0210*/ PRMT R5, R8, 0x7604, R5 ; + /*0220*/ PRMT R7, RZ, 0x4, R5.reuse ; + /*0230*/ PRMT R8, RZ, 0x5, R5 ; + /*0240*/ NOP ; + /*0250*/ BMSK R9, R7, R8 ; + /*0260*/ NOP ; + /*0270*/ SHF.L.U32 R7, R0, R7, RZ ; + /*0280*/ LOP3.LUT R7, R7, R9, R6, 0xe2, !PT ; + /*0290*/ STS [R10+0x10], R7 ; + /*02a0*/ EXIT ; + /*02b0*/ BRA 0x2b0; + +// == bmsk_clamp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ BMSK R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == bmsk_wrap +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ BMSK.W R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == szext_s +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SGXT R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == popc +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ POPC R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == popc64 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R6, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ POPC R0, R0 ; + /*0220*/ POPC R7, R6 ; + /*0230*/ IADD3 R7, R0, R7, RZ ; + /*0240*/ STS [R8+0x8], R7 ; + /*0250*/ EXIT ; + /*0260*/ BRA 0x260; + +// == clz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FLO.U32 R0, R2 ; + /*0200*/ IADD3 R7, -R0, 0x1f, RZ ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == clz64 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x8 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R5, [R8+0x4] ; + /*01f0*/ ISETP.NE.U32.AND P0, PT, R5, RZ, PT ; + /*0200*/ SEL R0, R0, R5, !P0 ; + /*0210*/ NOP ; + /*0220*/ FLO.U32 R0, R0 ; + /*0230*/ NOP ; + /*0240*/ IADD3 R6, -R0.reuse, 0x1f, RZ ; + /*0250*/ IADD3 R7, -R0, 0x3f, RZ ; + /*0260*/ @P0 IMAD.MOV R7, RZ, RZ, R6 ; + /*0270*/ STS [R8+0x8], R7 ; + /*0280*/ EXIT ; + /*0290*/ BRA 0x290; + +// == bfind_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FLO.U32 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == bfind_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FLO R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == bfind_shiftamt +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FLO.U32.SH R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == brev +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ BREV R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == fns +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x04d0 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x4 +// nop 0x0520 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0xc ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ IMAD R0, R3, 0x20, R0 ; + /*01c0*/ NOP ; + /*01d0*/ LDS R5, [R10] ; + /*01e0*/ LDS R6, [R10+0x4] ; + /*01f0*/ LDS R4, [R10+0x8] ; + /*0200*/ BSSY B0, 0x310 ; + /*0210*/ PLOP3.LUT P0, PT, PT, PT, PT, 0x8, 0x0 ; + /*0220*/ LOP3.LUT R4, R4, 0x1f, RZ, 0xc0, !PT ; + /*0230*/ ISETP.EQ.AND P1, PT, R4, RZ, PT ; + /*0240*/ @P1 BRA 0x2c0 ; + /*0250*/ ISETP.GT.AND P1, PT, R4, RZ, PT ; + /*0260*/ @P1 BRA 0x300 ; + /*0270*/ BREV R5, R5 ; + /*0280*/ PLOP3.LUT P0, PT, PT, PT, PT, 0x80, 0x0 ; + /*0290*/ IMAD.MOV R4, RZ, RZ, -R4 ; + /*02a0*/ IADD3 R6, -R6, 0x1f, RZ ; + /*02b0*/ BRA 0x300 ; + /*02c0*/ IMAD.MOV.U32 R3, RZ, RZ, 0x1 ; + /*02d0*/ IMAD.MOV.U32 R4, RZ, RZ, 0x1 ; + /*02e0*/ SHF.L.U32 R2, R3, R6, RZ ; + /*02f0*/ LOP3.LUT R5, R5, R2, RZ, 0xc0, !PT ; + /*0300*/ BSYNC B0 ; + /*0310*/ IMAD.MOV.U32 R3, RZ, RZ, -0x1 ; + /*0320*/ SHF.L.U32 R6, R3, R6, RZ ; + /*0330*/ LOP3.LUT R5, R5, R6, RZ, 0xc0, !PT ; + /*0340*/ PRMT R2, R5, 0x7710, RZ ; + /*0350*/ POPC R3, R5 ; + /*0360*/ POPC R7, R2 ; + /*0370*/ ISETP.LT.U32.AND P4, PT, R7, R4, PT ; + /*0380*/ IMAD.IADD R7, R4, 0x1, -R7 ; + /*0390*/ SEL R7, R7, R4, P4 ; + /*03a0*/ @P4 SHF.R.U32.HI R5, RZ, 0x10, R5 ; + /*03b0*/ PRMT R6, R5, 0x7770, RZ ; + /*03c0*/ POPC R6, R6 ; + /*03d0*/ ISETP.LT.U32.AND P3, PT, R6, R7, PT ; + /*03e0*/ @P3 SHF.R.U32.HI R5, RZ, 0x8, R5 ; + /*03f0*/ @P3 IMAD.IADD R7, R7, 0x1, -R6 ; + /*0400*/ SGXT.U32 R8, R5, 0x4 ; + /*0410*/ POPC R8, R8 ; + /*0420*/ ISETP.LT.U32.AND P2, PT, R8, R7, PT ; + /*0430*/ @P2 SHF.R.U32.HI R5, RZ, 0x4, R5 ; + /*0440*/ @P2 IMAD.IADD R7, R7, 0x1, -R8 ; + /*0450*/ SHF.R.U32.HI R9, RZ, 0x1, R5 ; + /*0460*/ SGXT.U32 R2, R5, 0x1 ; + /*0470*/ SGXT.U32 R9, R9, 0x1 ; + /*0480*/ IMAD.IADD R2, R2, 0x1, R9 ; + /*0490*/ ISETP.LT.U32.AND P1, PT, R2, R7, PT ; + /*04a0*/ @P1 SHF.R.U32.HI R5, RZ, 0x2, R5 ; + /*04b0*/ @P1 IMAD.IADD R7, R7, 0x1, -R2 ; + /*04c0*/ SGXT.U32 R2, R5, 0x1 ; + /*04d0*/ SEL R5, RZ, 0x10, !P4 ; + /*04e0*/ ISETP.LT.U32.AND P4, PT, R2, R7, PT ; + /*04f0*/ NOP ; + /*0500*/ @P3 IADD3 R5, R5, 0x8, RZ ; + /*0510*/ @P2 IADD3 R5, R5, 0x4, RZ ; + /*0520*/ @P1 IADD3 R5, R5, 0x2, RZ ; + /*0530*/ ISETP.LT.AND P1, PT, R3, R4, PT ; + /*0540*/ NOP ; + /*0550*/ @P4 IADD3 R5, R5, 0x1, RZ ; + /*0560*/ @P0 IADD3 R5, -R5, 0x1f, RZ ; + /*0570*/ SEL R5, R5, 0xffffffff, !P1 ; + /*0580*/ STS [R10+0xc], R5 ; + /*0590*/ EXIT ; + /*05a0*/ BRA 0x5a0; + +// == dp4a_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IDP.4A.U8.U8 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == dp4a_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IDP.4A.S8.S8 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == dp4a_su +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IDP.4A.S8.U8 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == dp2a_lo +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IDP.2A.LO.S16.S8 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == dp2a_hi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IDP.2A.HI.U16.U8 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shfl_idx +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHFL.IDX PT, R7, R0, R7, 0x1f ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == shfl_up +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R9, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LOP3.LUT R7, R5, 0x1f, RZ, 0xc0, !PT ; + /*0200*/ NOP ; + /*0210*/ SHFL.UP P0, R7, R0, R7, RZ ; + /*0220*/ SEL R9, RZ, 0x1, !P0 ; + /*0230*/ STS [R10+0x8], R7 ; + /*0240*/ STS [R10+0xc], R9 ; + /*0250*/ EXIT ; + /*0260*/ BRA 0x260; + +// == shfl_down +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R9, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LOP3.LUT R7, R5, 0x1f, RZ, 0xc0, !PT ; + /*0200*/ NOP ; + /*0210*/ SHFL.DOWN P0, R7, R0, R7, 0x1f ; + /*0220*/ SEL R9, RZ, 0x1, !P0 ; + /*0230*/ STS [R10+0x8], R7 ; + /*0240*/ STS [R10+0xc], R9 ; + /*0250*/ EXIT ; + /*0260*/ BRA 0x260; + +// == shfl_bfly +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHFL.BFLY PT, R7, R0, R7, 0x1f ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == shfl_bfly_imm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ SHFL.BFLY PT, R7, R2, 0x1, 0x1f ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == shfl_idx_seg +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHFL.IDX PT, R7, R0, R7, 0x1807 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == vote +// base register R14 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x4 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x10 +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R14, SR_TID.X ; + /*0010*/ IMAD.U32 R14, R14, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R14] ; + /*01e0*/ NOP ; + /*01f0*/ NOP ; + /*0200*/ LOP3.LUT P0, RZ, R2, 0x1, RZ, 0xc0, !PT ; + /*0210*/ VOTE.ANY P1, P0 ; + /*0220*/ VOTE.ALL P2, P0 ; + /*0230*/ VOTE.EQ P3, P0 ; + /*0240*/ VOTE.ANY R7, PT, P0 ; + /*0250*/ SEL R9, RZ, 0x1, !P1 ; + /*0260*/ SEL R11, RZ, 0x1, !P2 ; + /*0270*/ STS [R14+0x4], R7 ; + /*0280*/ SEL R13, RZ, 0x1, !P3 ; + /*0290*/ STS [R14+0x8], R9 ; + /*02a0*/ STS [R14+0xc], R11 ; + /*02b0*/ STS [R14+0x10], R13 ; + /*02c0*/ EXIT ; + /*02d0*/ BRA 0x2d0; + +// == activemask +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0190 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x01a0 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01b0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ S2R R3, SR_CTAID.X ; + /*01a0*/ VOTE.ANY R7, PT, PT ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ NOP ; + /*01e0*/ NOP ; + /*01f0*/ LDS R2, [R8] ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == match_any +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ LOP3.LUT R0, R2, 0x3, RZ, 0xc0, !PT ; + /*0200*/ MATCH.ANY R7, R0 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == redux_add +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ REDUX.SUM UR6, R2 ; + /*0200*/ MOV R7, UR6 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == redux_min_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ REDUX.MIN.S32 UR6, R2 ; + /*0200*/ MOV R7, UR6 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == redux_or +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ REDUX.OR UR6, R2 ; + /*0200*/ MOV R7, UR6 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == hmma_16816_f16 +// base register R14 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x20 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0230 (feeds only the removed addressing): MOV R5, 0x8 +// nop 0x0240 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R14, SR_TID.X ; + /*0010*/ IMAD.U32 R14, R14, 0x28, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R8, [R14] ; + /*01e0*/ LDS R9, [R14+0x4] ; + /*01f0*/ LDS R10, [R14+0x8] ; + /*0200*/ LDS R11, [R14+0xc] ; + /*0210*/ LDS R6, [R14+0x10] ; + /*0220*/ LDS R7, [R14+0x14] ; + /*0230*/ LDS R12, [R14+0x18] ; + /*0240*/ LDS R13, [R14+0x1c] ; + /*0250*/ NOP ; + /*0260*/ NOP ; + /*0270*/ HMMA.16816.F16 R6, R8, R6, R12 ; + /*0280*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*0290*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02a0*/ STS [R14+0x20], R6 ; + /*02b0*/ STS [R14+0x24], R7 ; + /*02c0*/ EXIT ; + /*02d0*/ BRA 0x2d0; + +// == hmma_16816_f32 +// base register R16 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x28 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0250 (feeds only the removed addressing): MOV R5, 0x10 +// nop 0x0260 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R16, SR_TID.X ; + /*0010*/ IMAD.U32 R16, R16, 0x38, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R8, [R16] ; + /*01e0*/ LDS R9, [R16+0x4] ; + /*01f0*/ LDS R10, [R16+0x8] ; + /*0200*/ LDS R11, [R16+0xc] ; + /*0210*/ LDS R6, [R16+0x10] ; + /*0220*/ LDS R7, [R16+0x14] ; + /*0230*/ LDS R12, [R16+0x18] ; + /*0240*/ LDS R13, [R16+0x1c] ; + /*0250*/ LDS R14, [R16+0x20] ; + /*0260*/ LDS R15, [R16+0x24] ; + /*0270*/ NOP ; + /*0280*/ NOP ; + /*0290*/ HMMA.16816.F32 R8, R8, R6, R12 ; + /*02a0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02b0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02c0*/ STS [R16+0x28], R8 ; + /*02d0*/ STS [R16+0x2c], R9 ; + /*02e0*/ STS [R16+0x30], R10 ; + /*02f0*/ STS [R16+0x34], R11 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == hmma_1688_f16 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x14 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0200 (feeds only the removed addressing): MOV R5, 0x8 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x1c, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R6, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R0, [R10+0x8] ; + /*0200*/ LDS R8, [R10+0xc] ; + /*0210*/ LDS R9, [R10+0x10] ; + /*0220*/ NOP ; + /*0230*/ NOP ; + /*0240*/ HMMA.1688.F16 R6, R6, R0, R8 ; + /*0250*/ STS [R10+0x14], R6 ; + /*0260*/ STS [R10+0x18], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == hmma_16816_bf16 +// base register R16 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x28 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0250 (feeds only the removed addressing): MOV R5, 0x10 +// nop 0x0260 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R16, SR_TID.X ; + /*0010*/ IMAD.U32 R16, R16, 0x38, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R8, [R16] ; + /*01e0*/ LDS R9, [R16+0x4] ; + /*01f0*/ LDS R10, [R16+0x8] ; + /*0200*/ LDS R11, [R16+0xc] ; + /*0210*/ LDS R6, [R16+0x10] ; + /*0220*/ LDS R7, [R16+0x14] ; + /*0230*/ LDS R12, [R16+0x18] ; + /*0240*/ LDS R13, [R16+0x1c] ; + /*0250*/ LDS R14, [R16+0x20] ; + /*0260*/ LDS R15, [R16+0x24] ; + /*0270*/ NOP ; + /*0280*/ NOP ; + /*0290*/ HMMA.16816.F32.BF16 R8, R8, R6, R12 ; + /*02a0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02b0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02c0*/ STS [R16+0x28], R8 ; + /*02d0*/ STS [R16+0x2c], R9 ; + /*02e0*/ STS [R16+0x30], R10 ; + /*02f0*/ STS [R16+0x34], R11 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == hmma_1688_tf32 +// base register R16 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x28 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0250 (feeds only the removed addressing): MOV R5, 0x10 +// nop 0x0260 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R16, SR_TID.X ; + /*0010*/ IMAD.U32 R16, R16, 0x38, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R8, [R16] ; + /*01e0*/ LDS R9, [R16+0x4] ; + /*01f0*/ LDS R10, [R16+0x8] ; + /*0200*/ LDS R11, [R16+0xc] ; + /*0210*/ LDS R6, [R16+0x10] ; + /*0220*/ LDS R7, [R16+0x14] ; + /*0230*/ LDS R12, [R16+0x18] ; + /*0240*/ LDS R13, [R16+0x1c] ; + /*0250*/ LDS R14, [R16+0x20] ; + /*0260*/ LDS R15, [R16+0x24] ; + /*0270*/ NOP ; + /*0280*/ NOP ; + /*0290*/ HMMA.1688.F32.TF32 R8, R8, R6, R12 ; + /*02a0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02b0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02c0*/ STS [R16+0x28], R8 ; + /*02d0*/ STS [R16+0x2c], R9 ; + /*02e0*/ STS [R16+0x30], R10 ; + /*02f0*/ STS [R16+0x34], R11 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == imma_16832_s8 +// base register R16 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x28 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0250 (feeds only the removed addressing): MOV R5, 0x10 +// nop 0x0260 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R16, SR_TID.X ; + /*0010*/ IMAD.U32 R16, R16, 0x38, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R8, [R16] ; + /*01e0*/ LDS R9, [R16+0x4] ; + /*01f0*/ LDS R10, [R16+0x8] ; + /*0200*/ LDS R11, [R16+0xc] ; + /*0210*/ LDS R6, [R16+0x10] ; + /*0220*/ LDS R7, [R16+0x14] ; + /*0230*/ LDS R12, [R16+0x18] ; + /*0240*/ LDS R13, [R16+0x1c] ; + /*0250*/ LDS R14, [R16+0x20] ; + /*0260*/ LDS R15, [R16+0x24] ; + /*0270*/ NOP ; + /*0280*/ NOP ; + /*0290*/ IMMA.16832.S8.S8 R8, R8.ROW, R6.COL, R12 ; + /*02a0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02b0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02c0*/ STS [R16+0x28], R8 ; + /*02d0*/ STS [R16+0x2c], R9 ; + /*02e0*/ STS [R16+0x30], R10 ; + /*02f0*/ STS [R16+0x34], R11 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == dadd +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R0, R3, R0, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R0, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 9.5367431640625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ LDS R7, [R10+0xc] ; + /*0210*/ NOP ; + /*0220*/ DADD R4, R4, R6 ; + /*0230*/ NOP ; + /*0240*/ STS [R10+0x10], R4 ; + /*0250*/ STS [R10+0x14], R5 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; + +// == dmul +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R0, R3, R0, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R0, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 9.5367431640625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ LDS R7, [R10+0xc] ; + /*0210*/ NOP ; + /*0220*/ DMUL R4, R4, R6 ; + /*0230*/ NOP ; + /*0240*/ STS [R10+0x10], R4 ; + /*0250*/ STS [R10+0x14], R5 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; diff --git a/dm/gfx/tests/ampere_orin/ampere_orin.u32.gz b/dm/gfx/tests/ampere_orin/ampere_orin.u32.gz new file mode 100644 index 0000000000..3482d102ad Binary files /dev/null and b/dm/gfx/tests/ampere_orin/ampere_orin.u32.gz differ diff --git a/dm/gfx/tests/ampere_orin/manifest.json b/dm/gfx/tests/ampere_orin/manifest.json new file mode 100644 index 0000000000..3d238c704f --- /dev/null +++ b/dm/gfx/tests/ampere_orin/manifest.json @@ -0,0 +1,3099 @@ +{ + "device": { + "gpu": "Orin", + "sm": "87", + "cuda_driver": "12.6", + "nvdisasm": "Build cuda_12.6.r12.6/compiler.34841621_0" + }, + "arch": "sm_87", + "ops": [ + { + "name": "fadd_rn", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rn.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 0, + 800 + ], + "rows": [ + 800, + 12288 + ] + }, + { + "name": "fmul_rn", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rn.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 13088, + 800 + ], + "rows": [ + 13888, + 12288 + ] + }, + { + "name": "ffma_rn", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rn.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 26176, + 800 + ], + "rows": [ + 26976, + 16384 + ] + }, + { + "name": "fadd_rz", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rz.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 43360, + 800 + ], + "rows": [ + 44160, + 12288 + ] + }, + { + "name": "fmul_rz", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rz.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 56448, + 800 + ], + "rows": [ + 57248, + 12288 + ] + }, + { + "name": "ffma_rz", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rz.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 69536, + 800 + ], + "rows": [ + 70336, + 16384 + ] + }, + { + "name": "fadd_rm", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rm.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 86720, + 800 + ], + "rows": [ + 87520, + 12288 + ] + }, + { + "name": "fmul_rm", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rm.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 99808, + 800 + ], + "rows": [ + 100608, + 12288 + ] + }, + { + "name": "ffma_rm", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rm.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 112896, + 800 + ], + "rows": [ + 113696, + 16384 + ] + }, + { + "name": "fadd_rp", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rp.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 130080, + 800 + ], + "rows": [ + 130880, + 12288 + ] + }, + { + "name": "fmul_rp", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rp.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 143168, + 800 + ], + "rows": [ + 143968, + 12288 + ] + }, + { + "name": "ffma_rp", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rp.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 156256, + 800 + ], + "rows": [ + 157056, + 16384 + ] + }, + { + "name": "fadd_ftz", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rn.ftz.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 173440, + 800 + ], + "rows": [ + 174240, + 12288 + ] + }, + { + "name": "fadd_sat", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rn.sat.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 186528, + 800 + ], + "rows": [ + 187328, + 12288 + ] + }, + { + "name": "fsub", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; sub.rn.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 199616, + 800 + ], + "rows": [ + 200416, + 12288 + ] + }, + { + "name": "fmul_ftz", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rn.ftz.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 212704, + 800 + ], + "rows": [ + 213504, + 12288 + ] + }, + { + "name": "fmul_sat", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rn.sat.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 225792, + 800 + ], + "rows": [ + 226592, + 12288 + ] + }, + { + "name": "ffma_ftz", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rn.ftz.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 238880, + 800 + ], + "rows": [ + 239680, + 16384 + ] + }, + { + "name": "ffma_sat", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rn.sat.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 256064, + 800 + ], + "rows": [ + 256864, + 16384 + ] + }, + { + "name": "ffma_ftz_sat", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rn.ftz.sat.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 273248, + 800 + ], + "rows": [ + 274048, + 16384 + ] + }, + { + "name": "fadd_abs_neg", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; abs.f32 %f2, %f0; neg.f32 %f3, %f1; add.rn.f32 %f4, %f2, %f3; mov.b32 %o0, %f4;", + "prog": [ + 290432, + 800 + ], + "rows": [ + 291232, + 12288 + ] + }, + { + "name": "fmul_by_imm", + "family": "f32 arith", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mul.rn.f32 %f2, %f0, 0f3EAAAAAB; mov.b32 %o0, %f2;", + "prog": [ + 303520, + 800 + ], + "rows": [ + 304320, + 8192 + ] + }, + { + "name": "fmin", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; min.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 312512, + 800 + ], + "rows": [ + 313312, + 12288 + ] + }, + { + "name": "fmax", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; max.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 325600, + 800 + ], + "rows": [ + 326400, + 12288 + ] + }, + { + "name": "fmin_ftz", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; min.ftz.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 338688, + 800 + ], + "rows": [ + 339488, + 12288 + ] + }, + { + "name": "fmax_nan", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; max.NaN.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 351776, + 800 + ], + "rows": [ + 352576, + 12288 + ] + }, + { + "name": "fmin_nan", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; min.NaN.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 364864, + 800 + ], + "rows": [ + 365664, + 12288 + ] + }, + { + "name": "fcopysign", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; copysign.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 377952, + 800 + ], + "rows": [ + 378752, + 12288 + ] + }, + { + "name": "fabs", + "family": "f32 arith", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; abs.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 391040, + 800 + ], + "rows": [ + 391840, + 8192 + ] + }, + { + "name": "fneg", + "family": "f32 arith", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; neg.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 400032, + 800 + ], + "rows": [ + 400832, + 8192 + ] + }, + { + "name": "fsetp", + "family": "f32 compare", + "ins": "ff", + "outs": "uuuuuuuuuuuuuu", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; setp.eq.f32 %p0, %f0, %f1; selp.u32 %o0, 1, 0, %p0; setp.ne.f32 %p0, %f0, %f1; selp.u32 %o1, 1, 0, %p0; setp.lt.f32 %p0, %f0, %f1; selp.u32 %o2, 1, 0, %p0; setp.le.f32 %p0, %f0, %f1; selp.u32 %o3, 1, 0, %p0; setp.gt.f32 %p0, %f0, %f1; selp.u32 %o4, 1, 0, %p0; setp.ge.f32 %p0, %f0, %f1; selp.u32 %o5, 1, 0, %p0; setp.equ.f32 %p0, %f0, %f1; selp.u32 %o6, 1, 0, %p0; setp.neu.f32 %p0, %f0, %f1; selp.u32 %o7, 1, 0, %p0; setp.ltu.f32 %p0, %f0, %f1; selp.u32 %o8, 1, 0, %p0; setp.leu.f32 %p0, %f0, %f1; selp.u32 %o9, 1, 0, %p0; setp.gtu.f32 %p0, %f0, %f1; selp.u32 %o10, 1, 0, %p0; setp.geu.f32 %p0, %f0, %f1; selp.u32 %o11, 1, 0, %p0; setp.num.f32 %p0, %f0, %f1; selp.u32 %o12, 1, 0, %p0; setp.nan.f32 %p0, %f0, %f1; selp.u32 %o13, 1, 0, %p0;", + "prog": [ + 409024, + 1312 + ], + "rows": [ + 410336, + 65536 + ] + }, + { + "name": "fsetp_ftz", + "family": "f32 compare", + "ins": "ff", + "outs": "uuuu", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; setp.eq.ftz.f32 %p0, %f0, %f1; selp.u32 %o0, 1, 0, %p0; setp.lt.ftz.f32 %p0, %f0, %f1; selp.u32 %o1, 1, 0, %p0; setp.gtu.ftz.f32 %p0, %f0, %f1; selp.u32 %o2, 1, 0, %p0; setp.ne.ftz.f32 %p0, %f0, %f1; selp.u32 %o3, 1, 0, %p0;", + "prog": [ + 475872, + 928 + ], + "rows": [ + 476800, + 24576 + ] + }, + { + "name": "fset_bf", + "family": "f32 compare", + "ins": "ff", + "outs": "ff", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; set.lt.f32.f32 %f2, %f0, %f1; set.neu.f32.f32 %f3, %f0, %f1; mov.b32 %o0, %f2; mov.b32 %o1, %f3;", + "prog": [ + 501376, + 800 + ], + "rows": [ + 502176, + 16384 + ] + }, + { + "name": "fset_u32", + "family": "f32 compare", + "ins": "ff", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; set.ge.u32.f32 %o0, %f0, %f1; set.nan.u32.f32 %o1, %f0, %f1;", + "prog": [ + 518560, + 800 + ], + "rows": [ + 519360, + 16384 + ] + }, + { + "name": "ftestp", + "family": "f32 compare", + "ins": "f", + "outs": "uuuuu", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; testp.finite.f32 %p0, %f0; selp.u32 %o0, 1, 0, %p0; testp.infinite.f32 %p0, %f0; selp.u32 %o1, 1, 0, %p0; testp.number.f32 %p0, %f0; selp.u32 %o2, 1, 0, %p0; testp.normal.f32 %p0, %f0; selp.u32 %o3, 1, 0, %p0; testp.subnormal.f32 %p0, %f0; selp.u32 %o4, 1, 0, %p0;", + "prog": [ + 535744, + 1056 + ], + "rows": [ + 536800, + 24576 + ] + }, + { + "name": "fsel_cmp", + "family": "f32 compare", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; setp.gt.f32 %p0, %f0, 0f00000000; selp.f32 %f2, %f1, %f3, %p0; mov.b32 %o0, %f2;", + "prog": [ + 561376, + 800 + ], + "rows": [ + 562176, + 16384 + ] + }, + { + "name": "mufu_ex2", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; ex2.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 578560, + 800 + ], + "rows": [ + 579360, + 8192 + ] + }, + { + "name": "mufu_ex2_ftz", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; ex2.approx.ftz.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 587552, + 800 + ], + "rows": [ + 588352, + 8192 + ] + }, + { + "name": "mufu_lg2", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; lg2.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 596544, + 800 + ], + "rows": [ + 597344, + 8192 + ] + }, + { + "name": "mufu_lg2_ftz", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; lg2.approx.ftz.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 605536, + 800 + ], + "rows": [ + 606336, + 8192 + ] + }, + { + "name": "mufu_rcp", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; rcp.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 614528, + 800 + ], + "rows": [ + 615328, + 8192 + ] + }, + { + "name": "mufu_rcp_ftz", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; rcp.approx.ftz.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 623520, + 800 + ], + "rows": [ + 624320, + 8192 + ] + }, + { + "name": "mufu_rsqrt", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; rsqrt.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 632512, + 800 + ], + "rows": [ + 633312, + 8192 + ] + }, + { + "name": "mufu_rsqrt_ftz", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; rsqrt.approx.ftz.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 641504, + 800 + ], + "rows": [ + 642304, + 8192 + ] + }, + { + "name": "mufu_sqrt", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; sqrt.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 650496, + 800 + ], + "rows": [ + 651296, + 8192 + ] + }, + { + "name": "mufu_sqrt_ftz", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; sqrt.approx.ftz.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 659488, + 800 + ], + "rows": [ + 660288, + 8192 + ] + }, + { + "name": "mufu_sin", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; sin.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 668480, + 800 + ], + "rows": [ + 669280, + 8192 + ] + }, + { + "name": "mufu_cos", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cos.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 677472, + 800 + ], + "rows": [ + 678272, + 8192 + ] + }, + { + "name": "mufu_tanh", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; tanh.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 686464, + 800 + ], + "rows": [ + 687264, + 8192 + ] + }, + { + "name": "mufu_ex2_f16x2", + "family": "mufu", + "ins": "h", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; ex2.approx.f16x2 %x2, %x0; mov.b32 %o0, %x2;", + "prog": [ + 695456, + 800 + ], + "rows": [ + 696256, + 8192 + ] + }, + { + "name": "mufu_tanh_f16x2", + "family": "mufu", + "ins": "h", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; tanh.approx.f16x2 %x2, %x0; mov.b32 %o0, %x2;", + "prog": [ + 704448, + 800 + ], + "rows": [ + 705248, + 8192 + ] + }, + { + "name": "fdiv_rn", + "family": "f32 precise", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; div.rn.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 713440, + 928 + ], + "rows": [ + 714368, + 12288 + ] + }, + { + "name": "fdiv_full", + "family": "f32 precise", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; div.full.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 726656, + 928 + ], + "rows": [ + 727584, + 12288 + ] + }, + { + "name": "fdiv_approx", + "family": "f32 precise", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; div.approx.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 739872, + 800 + ], + "rows": [ + 740672, + 12288 + ] + }, + { + "name": "frcp_rn", + "family": "f32 precise", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; rcp.rn.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 752960, + 928 + ], + "rows": [ + 753888, + 8192 + ] + }, + { + "name": "fsqrt_rn", + "family": "f32 precise", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; sqrt.rn.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 762080, + 928 + ], + "rows": [ + 763008, + 8192 + ] + }, + { + "name": "f2i_s32_rni", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rni.s32.f32 %o0, %f0;", + "prog": [ + 771200, + 800 + ], + "rows": [ + 772000, + 8192 + ] + }, + { + "name": "frnd_rni", + "family": "convert", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rni.f32.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 780192, + 800 + ], + "rows": [ + 780992, + 8192 + ] + }, + { + "name": "f2i_s32_rzi", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.s32.f32 %o0, %f0;", + "prog": [ + 789184, + 800 + ], + "rows": [ + 789984, + 8192 + ] + }, + { + "name": "frnd_rzi", + "family": "convert", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.f32.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 798176, + 800 + ], + "rows": [ + 798976, + 8192 + ] + }, + { + "name": "f2i_s32_rmi", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rmi.s32.f32 %o0, %f0;", + "prog": [ + 807168, + 800 + ], + "rows": [ + 807968, + 8192 + ] + }, + { + "name": "frnd_rmi", + "family": "convert", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rmi.f32.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 816160, + 800 + ], + "rows": [ + 816960, + 8192 + ] + }, + { + "name": "f2i_s32_rpi", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rpi.s32.f32 %o0, %f0;", + "prog": [ + 825152, + 800 + ], + "rows": [ + 825952, + 8192 + ] + }, + { + "name": "frnd_rpi", + "family": "convert", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rpi.f32.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 834144, + 800 + ], + "rows": [ + 834944, + 8192 + ] + }, + { + "name": "f2i_u32_rzi", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.u32.f32 %o0, %f0;", + "prog": [ + 843136, + 800 + ], + "rows": [ + 843936, + 8192 + ] + }, + { + "name": "f2i_s32_ftz", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.ftz.s32.f32 %o0, %f0;", + "prog": [ + 852128, + 800 + ], + "rows": [ + 852928, + 8192 + ] + }, + { + "name": "f2i_s16_sat", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.sat.s16.f32 %w0, %f0; cvt.s32.s16 %o0, %w0;", + "prog": [ + 861120, + 800 + ], + "rows": [ + 861920, + 8192 + ] + }, + { + "name": "f2i_u8_sat", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rni.sat.u8.f32 %w0, %f0; cvt.u32.u8 %o0, %w0;", + "prog": [ + 870112, + 800 + ], + "rows": [ + 870912, + 8192 + ] + }, + { + "name": "f2i_s64", + "family": "convert", + "ins": "f", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.s64.f32 %d0, %f0; mov.b64 {%o0, %o1}, %d0;", + "prog": [ + 879104, + 800 + ], + "rows": [ + 879904, + 12288 + ] + }, + { + "name": "i2f_s32_rn", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.rn.f32.s32 %f2, %i0; mov.b32 %o0, %f2;", + "prog": [ + 892192, + 800 + ], + "rows": [ + 892992, + 8192 + ] + }, + { + "name": "i2f_s32_rz", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.rz.f32.s32 %f2, %i0; mov.b32 %o0, %f2;", + "prog": [ + 901184, + 800 + ], + "rows": [ + 901984, + 8192 + ] + }, + { + "name": "i2f_s32_rm", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.rm.f32.s32 %f2, %i0; mov.b32 %o0, %f2;", + "prog": [ + 910176, + 800 + ], + "rows": [ + 910976, + 8192 + ] + }, + { + "name": "i2f_s32_rp", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.rp.f32.s32 %f2, %i0; mov.b32 %o0, %f2;", + "prog": [ + 919168, + 800 + ], + "rows": [ + 919968, + 8192 + ] + }, + { + "name": "i2f_u32_rn", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.rn.f32.u32 %f2, %i0; mov.b32 %o0, %f2;", + "prog": [ + 928160, + 800 + ], + "rows": [ + 928960, + 8192 + ] + }, + { + "name": "i2f_s16", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.u16.u32 %w0, %i0; cvt.rn.f32.s16 %f2, %w0; mov.b32 %o0, %f2;", + "prog": [ + 937152, + 800 + ], + "rows": [ + 937952, + 8192 + ] + }, + { + "name": "i2f_s64", + "family": "convert", + "ins": "uu", + "outs": "f", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; cvt.rn.f32.s64 %f2, %d0; mov.b32 %o0, %f2;", + "prog": [ + 946144, + 800 + ], + "rows": [ + 946944, + 12288 + ] + }, + { + "name": "f2f_f16_rn", + "family": "convert", + "ins": "f", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rn.f16.f32 %y0, %f0; mov.b16 %w0, %y0; cvt.u32.u16 %o0, %w0;", + "prog": [ + 959232, + 800 + ], + "rows": [ + 960032, + 8192 + ] + }, + { + "name": "f2f_f16_rz", + "family": "convert", + "ins": "f", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rz.f16.f32 %y0, %f0; mov.b16 %w0, %y0; cvt.u32.u16 %o0, %w0;", + "prog": [ + 968224, + 800 + ], + "rows": [ + 969024, + 8192 + ] + }, + { + "name": "f2f_f32_from_f16", + "family": "convert", + "ins": "h", + "outs": "ff", + "cases": 1024, + "ptx": "mov.b32 {%y0, %y1}, %i0; cvt.f32.f16 %f2, %y0; cvt.f32.f16 %f3, %y1; mov.b32 %o0, %f2; mov.b32 %o1, %f3;", + "prog": [ + 977216, + 800 + ], + "rows": [ + 978016, + 12288 + ] + }, + { + "name": "f2fp_pack_f16x2", + "family": "convert", + "ins": "ff", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.f16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;", + "prog": [ + 990304, + 800 + ], + "rows": [ + 991104, + 12288 + ] + }, + { + "name": "f2fp_pack_f16x2_relu", + "family": "convert", + "ins": "ff", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.relu.f16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;", + "prog": [ + 1003392, + 800 + ], + "rows": [ + 1004192, + 12288 + ] + }, + { + "name": "f2fp_pack_bf16x2", + "family": "convert", + "ins": "ff", + "outs": "b", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.bf16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;", + "prog": [ + 1016480, + 800 + ], + "rows": [ + 1017280, + 12288 + ] + }, + { + "name": "f2f_tf32", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rna.tf32.f32 %o0, %f0;", + "prog": [ + 1029568, + 800 + ], + "rows": [ + 1030368, + 8192 + ] + }, + { + "name": "i2i_s8_sat", + "family": "convert", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "cvt.sat.s8.s32 %w0, %i0; cvt.s32.s8 %o0, %w0;", + "prog": [ + 1038560, + 800 + ], + "rows": [ + 1039360, + 8192 + ] + }, + { + "name": "i2i_u16_sat", + "family": "convert", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "cvt.sat.u16.s32 %w0, %i0; cvt.u32.u16 %o0, %w0;", + "prog": [ + 1047552, + 800 + ], + "rows": [ + 1048352, + 8192 + ] + }, + { + "name": "sext_s8", + "family": "convert", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "cvt.s32.s8 %o0, %i0;", + "prog": [ + 1056544, + 800 + ], + "rows": [ + 1057344, + 8192 + ] + }, + { + "name": "hadd2", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; add.rn.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1065536, + 800 + ], + "rows": [ + 1066336, + 12288 + ] + }, + { + "name": "hadd2_ftz", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; add.rn.ftz.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1078624, + 800 + ], + "rows": [ + 1079424, + 12288 + ] + }, + { + "name": "hadd2_sat", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; add.rn.sat.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1091712, + 800 + ], + "rows": [ + 1092512, + 12288 + ] + }, + { + "name": "hsub2", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; sub.rn.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1104800, + 800 + ], + "rows": [ + 1105600, + 12288 + ] + }, + { + "name": "hmul2", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mul.rn.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1117888, + 800 + ], + "rows": [ + 1118688, + 12288 + ] + }, + { + "name": "hmul2_ftz", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mul.rn.ftz.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1130976, + 800 + ], + "rows": [ + 1131776, + 12288 + ] + }, + { + "name": "hfma2", + "family": "f16x2", + "ins": "hhh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; fma.rn.f16x2 %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;", + "prog": [ + 1144064, + 800 + ], + "rows": [ + 1144864, + 16384 + ] + }, + { + "name": "hfma2_ftz", + "family": "f16x2", + "ins": "hhh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; fma.rn.ftz.f16x2 %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;", + "prog": [ + 1161248, + 800 + ], + "rows": [ + 1162048, + 16384 + ] + }, + { + "name": "hfma2_sat", + "family": "f16x2", + "ins": "hhh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; fma.rn.sat.f16x2 %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;", + "prog": [ + 1178432, + 800 + ], + "rows": [ + 1179232, + 16384 + ] + }, + { + "name": "hfma2_relu", + "family": "f16x2", + "ins": "hhh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; fma.rn.relu.f16x2 %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;", + "prog": [ + 1195616, + 800 + ], + "rows": [ + 1196416, + 16384 + ] + }, + { + "name": "hmin2", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; min.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1212800, + 800 + ], + "rows": [ + 1213600, + 12288 + ] + }, + { + "name": "hmax2", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; max.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1225888, + 800 + ], + "rows": [ + 1226688, + 12288 + ] + }, + { + "name": "hmin2_nan", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; min.NaN.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1238976, + 800 + ], + "rows": [ + 1239776, + 12288 + ] + }, + { + "name": "hneg2", + "family": "f16x2", + "ins": "h", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; neg.f16x2 %x2, %x0; mov.b32 %o0, %x2;", + "prog": [ + 1252064, + 800 + ], + "rows": [ + 1252864, + 8192 + ] + }, + { + "name": "habs2", + "family": "f16x2", + "ins": "h", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; abs.f16x2 %x2, %x0; mov.b32 %o0, %x2;", + "prog": [ + 1261056, + 800 + ], + "rows": [ + 1261856, + 8192 + ] + }, + { + "name": "hadd2_imm", + "family": "f16x2", + "ins": "h", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, 0x3C00BC00; add.rn.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1270048, + 800 + ], + "rows": [ + 1270848, + 8192 + ] + }, + { + "name": "hadd_f16", + "family": "f16x2", + "ins": "hh", + "outs": "hh", + "cases": 1024, + "ptx": "mov.b32 {%y0, %y1}, %i0; mov.b32 {%y2, %y3}, %i1; add.rn.f16 %y4, %y0, %y2; mul.rn.f16 %y5, %y1, %y3; mov.b16 %w0, %y4; cvt.u32.u16 %o0, %w0; mov.b16 %w1, %y5; cvt.u32.u16 %o1, %w1;", + "prog": [ + 1279040, + 800 + ], + "rows": [ + 1279840, + 16384 + ] + }, + { + "name": "hsetp2", + "family": "f16x2", + "ins": "hh", + "outs": "uuuuuuuu", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; setp.lt.f16x2 %p0|%p1, %x0, %x1; selp.u32 %o0, 1, 0, %p0; selp.u32 %o1, 1, 0, %p1; setp.eq.f16x2 %p0|%p1, %x0, %x1; selp.u32 %o2, 1, 0, %p0; selp.u32 %o3, 1, 0, %p1; setp.gtu.f16x2 %p0|%p1, %x0, %x1; selp.u32 %o4, 1, 0, %p0; selp.u32 %o5, 1, 0, %p1; setp.nan.f16x2 %p0|%p1, %x0, %x1; selp.u32 %o6, 1, 0, %p0; selp.u32 %o7, 1, 0, %p1;", + "prog": [ + 1296224, + 1056 + ], + "rows": [ + 1297280, + 40960 + ] + }, + { + "name": "hset2", + "family": "f16x2", + "ins": "hh", + "outs": "hu", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; set.lt.f16x2.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2; set.geu.u32.f16x2 %o1, %x0, %x1;", + "prog": [ + 1338240, + 800 + ], + "rows": [ + 1339040, + 16384 + ] + }, + { + "name": "bfma2", + "family": "bf16x2", + "ins": "bbb", + "outs": "b", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; fma.rn.bf16x2 %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;", + "prog": [ + 1355424, + 800 + ], + "rows": [ + 1356224, + 16384 + ] + }, + { + "name": "bmin2", + "family": "bf16x2", + "ins": "bb", + "outs": "b", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; min.bf16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1372608, + 800 + ], + "rows": [ + 1373408, + 12288 + ] + }, + { + "name": "bmax2_nan", + "family": "bf16x2", + "ins": "bb", + "outs": "b", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; max.NaN.bf16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1385696, + 800 + ], + "rows": [ + 1386496, + 12288 + ] + }, + { + "name": "iadd", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "add.u32 %o0, %i0, %i1;", + "prog": [ + 1398784, + 800 + ], + "rows": [ + 1399584, + 12288 + ] + }, + { + "name": "iadd3", + "family": "int", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "add.u32 %t0, %i0, %i1; sub.u32 %o0, %t0, %i2;", + "prog": [ + 1411872, + 800 + ], + "rows": [ + 1412672, + 16384 + ] + }, + { + "name": "iadd_sat", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "add.sat.s32 %o0, %i0, %i1;", + "prog": [ + 1429056, + 800 + ], + "rows": [ + 1429856, + 12288 + ] + }, + { + "name": "isub_sat", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "sub.sat.s32 %o0, %i0, %i1;", + "prog": [ + 1442144, + 800 + ], + "rows": [ + 1442944, + 12288 + ] + }, + { + "name": "iadd64", + "family": "int", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; add.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;", + "prog": [ + 1455232, + 800 + ], + "rows": [ + 1456032, + 24576 + ] + }, + { + "name": "isub64", + "family": "int", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; sub.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;", + "prog": [ + 1480608, + 800 + ], + "rows": [ + 1481408, + 24576 + ] + }, + { + "name": "imul_lo", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mul.lo.u32 %o0, %i0, %i1;", + "prog": [ + 1505984, + 800 + ], + "rows": [ + 1506784, + 12288 + ] + }, + { + "name": "imul_hi_u32", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mul.hi.u32 %o0, %i0, %i1;", + "prog": [ + 1519072, + 800 + ], + "rows": [ + 1519872, + 12288 + ] + }, + { + "name": "imul_hi_s32", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mul.hi.s32 %o0, %i0, %i1;", + "prog": [ + 1532160, + 800 + ], + "rows": [ + 1532960, + 12288 + ] + }, + { + "name": "imul_wide_s32", + "family": "int", + "ins": "uu", + "outs": "uu", + "cases": 1024, + "ptx": "mul.wide.s32 %d0, %i0, %i1; mov.b64 {%o0, %o1}, %d0;", + "prog": [ + 1545248, + 800 + ], + "rows": [ + 1546048, + 16384 + ] + }, + { + "name": "imul_wide_u32", + "family": "int", + "ins": "uu", + "outs": "uu", + "cases": 1024, + "ptx": "mul.wide.u32 %d0, %i0, %i1; mov.b64 {%o0, %o1}, %d0;", + "prog": [ + 1562432, + 800 + ], + "rows": [ + 1563232, + 16384 + ] + }, + { + "name": "imad_lo", + "family": "int", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "mad.lo.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 1579616, + 800 + ], + "rows": [ + 1580416, + 16384 + ] + }, + { + "name": "imad_hi_s32", + "family": "int", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "mad.hi.s32 %o0, %i0, %i1, %i2;", + "prog": [ + 1596800, + 800 + ], + "rows": [ + 1597600, + 16384 + ] + }, + { + "name": "imul64", + "family": "int", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; mul.lo.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;", + "prog": [ + 1613984, + 928 + ], + "rows": [ + 1614912, + 24576 + ] + }, + { + "name": "imul24", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mul24.lo.s32 %o0, %i0, %i1;", + "prog": [ + 1639488, + 800 + ], + "rows": [ + 1640288, + 12288 + ] + }, + { + "name": "imin_s32", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "min.s32 %o0, %i0, %i1;", + "prog": [ + 1652576, + 800 + ], + "rows": [ + 1653376, + 12288 + ] + }, + { + "name": "imax_u32", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "max.u32 %o0, %i0, %i1;", + "prog": [ + 1665664, + 800 + ], + "rows": [ + 1666464, + 12288 + ] + }, + { + "name": "imin_s64", + "family": "int", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; min.s64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;", + "prog": [ + 1678752, + 928 + ], + "rows": [ + 1679680, + 24576 + ] + }, + { + "name": "iabs", + "family": "int", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "abs.s32 %o0, %i0;", + "prog": [ + 1704256, + 800 + ], + "rows": [ + 1705056, + 8192 + ] + }, + { + "name": "ineg", + "family": "int", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "neg.s32 %o0, %i0;", + "prog": [ + 1713248, + 800 + ], + "rows": [ + 1714048, + 8192 + ] + }, + { + "name": "isad", + "family": "int", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "sad.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 1722240, + 800 + ], + "rows": [ + 1723040, + 16384 + ] + }, + { + "name": "idiv_u32", + "family": "int div", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "div.u32 %o0, %i0, %i1;", + "prog": [ + 1739424, + 1056 + ], + "rows": [ + 1740480, + 12288 + ] + }, + { + "name": "irem_u32", + "family": "int div", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "rem.u32 %o0, %i0, %i1;", + "prog": [ + 1752768, + 1056 + ], + "rows": [ + 1753824, + 12288 + ] + }, + { + "name": "idiv_s32", + "family": "int div", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "div.s32 %o0, %i0, %i1;", + "prog": [ + 1766112, + 1184 + ], + "rows": [ + 1767296, + 12288 + ] + }, + { + "name": "irem_s32", + "family": "int div", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "rem.s32 %o0, %i0, %i1;", + "prog": [ + 1779584, + 1184 + ], + "rows": [ + 1780768, + 12288 + ] + }, + { + "name": "idiv_u32_by_7", + "family": "int div", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "div.u32 %o0, %i0, 7;", + "prog": [ + 1793056, + 928 + ], + "rows": [ + 1793984, + 8192 + ] + }, + { + "name": "isetp", + "family": "int compare", + "ins": "uu", + "outs": "uuuuuu", + "cases": 1024, + "ptx": "setp.lt.s32 %p0, %i0, %i1; selp.u32 %o0, 1, 0, %p0; setp.lt.u32 %p0, %i0, %i1; selp.u32 %o1, 1, 0, %p0; setp.ge.s32 %p0, %i0, %i1; selp.u32 %o2, 1, 0, %p0; setp.hi.u32 %p0, %i0, %i1; selp.u32 %o3, 1, 0, %p0; setp.eq.s32 %p0, %i0, %i1; selp.u32 %o4, 1, 0, %p0; setp.ne.u32 %p0, %i0, %i1; selp.u32 %o5, 1, 0, %p0;", + "prog": [ + 1802176, + 1056 + ], + "rows": [ + 1803232, + 32768 + ] + }, + { + "name": "isetp_and", + "family": "int compare", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "setp.lt.s32 %p1, %i2, %i3; setp.lt.and.s32 %p0, %i0, %i1, %p1; selp.u32 %o0, 1, 0, %p0; setp.ge.or.u32 %p2, %i0, %i1, %p1; selp.u32 %o1, 1, 0, %p2;", + "prog": [ + 1836000, + 928 + ], + "rows": [ + 1836928, + 24576 + ] + }, + { + "name": "isetp64", + "family": "int compare", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; setp.lt.s64 %p0, %d0, %d1; selp.u32 %o0, 1, 0, %p0; setp.lt.u64 %p1, %d0, %d1; selp.u32 %o1, 1, 0, %p1;", + "prog": [ + 1861504, + 928 + ], + "rows": [ + 1862432, + 24576 + ] + }, + { + "name": "plop3", + "family": "int compare", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "setp.ne.u32 %p0, %i0, 0; setp.ne.u32 %p1, %i1, 0; setp.ne.u32 %p2, %i2, 0; xor.pred %p3, %p0, %p1; or.pred %p3, %p3, %p2; not.pred %p3, %p3; selp.u32 %o0, 1, 0, %p3;", + "prog": [ + 1887008, + 800 + ], + "rows": [ + 1887808, + 16384 + ] + }, + { + "name": "shl", + "family": "bits", + "ins": "us", + "outs": "u", + "cases": 1024, + "ptx": "shl.b32 %o0, %i0, %i1;", + "prog": [ + 1904192, + 800 + ], + "rows": [ + 1904992, + 12288 + ] + }, + { + "name": "shr_u32", + "family": "bits", + "ins": "us", + "outs": "u", + "cases": 1024, + "ptx": "shr.u32 %o0, %i0, %i1;", + "prog": [ + 1917280, + 800 + ], + "rows": [ + 1918080, + 12288 + ] + }, + { + "name": "shr_s32", + "family": "bits", + "ins": "us", + "outs": "u", + "cases": 1024, + "ptx": "shr.s32 %o0, %i0, %i1;", + "prog": [ + 1930368, + 800 + ], + "rows": [ + 1931168, + 12288 + ] + }, + { + "name": "shl_imm", + "family": "bits", + "ins": "u", + "outs": "uu", + "cases": 1024, + "ptx": "shl.b32 %o0, %i0, 7; shr.s32 %o1, %i0, 13;", + "prog": [ + 1943456, + 800 + ], + "rows": [ + 1944256, + 12288 + ] + }, + { + "name": "shf_l_wrap", + "family": "bits", + "ins": "uus", + "outs": "u", + "cases": 1024, + "ptx": "shf.l.wrap.b32 %o0, %i0, %i1, %i2;", + "prog": [ + 1956544, + 800 + ], + "rows": [ + 1957344, + 16384 + ] + }, + { + "name": "shf_l_clamp", + "family": "bits", + "ins": "uus", + "outs": "u", + "cases": 1024, + "ptx": "shf.l.clamp.b32 %o0, %i0, %i1, %i2;", + "prog": [ + 1973728, + 800 + ], + "rows": [ + 1974528, + 16384 + ] + }, + { + "name": "shf_r_wrap", + "family": "bits", + "ins": "uus", + "outs": "u", + "cases": 1024, + "ptx": "shf.r.wrap.b32 %o0, %i0, %i1, %i2;", + "prog": [ + 1990912, + 800 + ], + "rows": [ + 1991712, + 16384 + ] + }, + { + "name": "shf_r_clamp", + "family": "bits", + "ins": "uus", + "outs": "u", + "cases": 1024, + "ptx": "shf.r.clamp.b32 %o0, %i0, %i1, %i2;", + "prog": [ + 2008096, + 800 + ], + "rows": [ + 2008896, + 16384 + ] + }, + { + "name": "shl64", + "family": "bits", + "ins": "uus", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; shl.b64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;", + "prog": [ + 2025280, + 800 + ], + "rows": [ + 2026080, + 20480 + ] + }, + { + "name": "shr64_u", + "family": "bits", + "ins": "uus", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; shr.u64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;", + "prog": [ + 2046560, + 800 + ], + "rows": [ + 2047360, + 20480 + ] + }, + { + "name": "shr64_s", + "family": "bits", + "ins": "uus", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; shr.s64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;", + "prog": [ + 2067840, + 800 + ], + "rows": [ + 2068640, + 20480 + ] + }, + { + "name": "lop3", + "family": "bits", + "ins": "uuu", + "outs": "uuuuuuuuuu", + "cases": 1024, + "ptx": "lop3.b32 %o0, %i0, %i1, %i2, 0x96; lop3.b32 %o1, %i0, %i1, %i2, 0xe8; lop3.b32 %o2, %i0, %i1, %i2, 0xca; lop3.b32 %o3, %i0, %i1, %i2, 0x1; lop3.b32 %o4, %i0, %i1, %i2, 0x80; lop3.b32 %o5, %i0, %i1, %i2, 0x6a; lop3.b32 %o6, %i0, %i1, %i2, 0xf8; lop3.b32 %o7, %i0, %i1, %i2, 0x1e; lop3.b32 %o8, %i0, %i1, %i2, 0xb4; lop3.b32 %o9, %i0, %i1, %i2, 0x3c;", + "prog": [ + 2089120, + 1056 + ], + "rows": [ + 2090176, + 53248 + ] + }, + { + "name": "logic", + "family": "bits", + "ins": "uu", + "outs": "uuuu", + "cases": 1024, + "ptx": "and.b32 %o0, %i0, %i1; or.b32 %o1, %i0, %i1; xor.b32 %o2, %i0, %i1; not.b32 %t0, %i0; and.b32 %o3, %t0, %i1;", + "prog": [ + 2143424, + 928 + ], + "rows": [ + 2144352, + 24576 + ] + }, + { + "name": "prmt", + "family": "bits", + "ins": "uuu", + "outs": "uuuuuuu", + "cases": 1024, + "ptx": "prmt.b32 %o0, %i0, %i1, %i2; prmt.b32.f4e %o1, %i0, %i1, %i2; prmt.b32.b4e %o2, %i0, %i1, %i2; prmt.b32.rc8 %o3, %i0, %i1, %i2; prmt.b32.ecl %o4, %i0, %i1, %i2; prmt.b32.ecr %o5, %i0, %i1, %i2; prmt.b32.rc16 %o6, %i0, %i1, %i2;", + "prog": [ + 2168928, + 928 + ], + "rows": [ + 2169856, + 40960 + ] + }, + { + "name": "prmt_imm", + "family": "bits", + "ins": "uu", + "outs": "uuu", + "cases": 1024, + "ptx": "prmt.b32 %o0, %i0, %i1, 0x3210; prmt.b32 %o1, %i0, %i1, 0x5140; prmt.b32 %o2, %i0, %i1, 0x8880;", + "prog": [ + 2210816, + 800 + ], + "rows": [ + 2211616, + 20480 + ] + }, + { + "name": "bfe_u32", + "family": "bits", + "ins": "uss", + "outs": "u", + "cases": 1024, + "ptx": "bfe.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 2232096, + 800 + ], + "rows": [ + 2232896, + 16384 + ] + }, + { + "name": "bfe_s32", + "family": "bits", + "ins": "uss", + "outs": "u", + "cases": 1024, + "ptx": "bfe.s32 %o0, %i0, %i1, %i2;", + "prog": [ + 2249280, + 800 + ], + "rows": [ + 2250080, + 16384 + ] + }, + { + "name": "bfi", + "family": "bits", + "ins": "uuss", + "outs": "u", + "cases": 1024, + "ptx": "bfi.b32 %o0, %i0, %i1, %i2, %i3;", + "prog": [ + 2266464, + 928 + ], + "rows": [ + 2267392, + 20480 + ] + }, + { + "name": "bmsk_clamp", + "family": "bits", + "ins": "ss", + "outs": "u", + "cases": 1024, + "ptx": "bmsk.clamp.b32 %o0, %i0, %i1;", + "prog": [ + 2287872, + 800 + ], + "rows": [ + 2288672, + 12288 + ] + }, + { + "name": "bmsk_wrap", + "family": "bits", + "ins": "ss", + "outs": "u", + "cases": 1024, + "ptx": "bmsk.wrap.b32 %o0, %i0, %i1;", + "prog": [ + 2300960, + 800 + ], + "rows": [ + 2301760, + 12288 + ] + }, + { + "name": "szext_s", + "family": "bits", + "ins": "us", + "outs": "u", + "cases": 1024, + "ptx": "szext.clamp.s32 %o0, %i0, %i1;", + "prog": [ + 2314048, + 800 + ], + "rows": [ + 2314848, + 12288 + ] + }, + { + "name": "popc", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "popc.b32 %o0, %i0;", + "prog": [ + 2327136, + 800 + ], + "rows": [ + 2327936, + 8192 + ] + }, + { + "name": "popc64", + "family": "bits", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; popc.b64 %o0, %d0;", + "prog": [ + 2336128, + 800 + ], + "rows": [ + 2336928, + 12288 + ] + }, + { + "name": "clz", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "clz.b32 %o0, %i0;", + "prog": [ + 2349216, + 800 + ], + "rows": [ + 2350016, + 8192 + ] + }, + { + "name": "clz64", + "family": "bits", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; clz.b64 %o0, %d0;", + "prog": [ + 2358208, + 800 + ], + "rows": [ + 2359008, + 12288 + ] + }, + { + "name": "bfind_u32", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "bfind.u32 %o0, %i0;", + "prog": [ + 2371296, + 800 + ], + "rows": [ + 2372096, + 8192 + ] + }, + { + "name": "bfind_s32", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "bfind.s32 %o0, %i0;", + "prog": [ + 2380288, + 800 + ], + "rows": [ + 2381088, + 8192 + ] + }, + { + "name": "bfind_shiftamt", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "bfind.shiftamt.u32 %o0, %i0;", + "prog": [ + 2389280, + 800 + ], + "rows": [ + 2390080, + 8192 + ] + }, + { + "name": "brev", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "brev.b32 %o0, %i0;", + "prog": [ + 2398272, + 800 + ], + "rows": [ + 2399072, + 8192 + ] + }, + { + "name": "fns", + "family": "bits", + "ins": "uus", + "outs": "u", + "cases": 1024, + "ptx": "and.b32 %t0, %i2, 31; fns.b32 %o0, %i0, %i1, %t0;", + "prog": [ + 2407264, + 1696 + ], + "rows": [ + 2408960, + 16384 + ] + }, + { + "name": "dp4a_u32", + "family": "bits", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "dp4a.u32.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 2425344, + 800 + ], + "rows": [ + 2426144, + 16384 + ] + }, + { + "name": "dp4a_s32", + "family": "bits", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "dp4a.s32.s32 %o0, %i0, %i1, %i2;", + "prog": [ + 2442528, + 800 + ], + "rows": [ + 2443328, + 16384 + ] + }, + { + "name": "dp4a_su", + "family": "bits", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "dp4a.s32.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 2459712, + 800 + ], + "rows": [ + 2460512, + 16384 + ] + }, + { + "name": "dp2a_lo", + "family": "bits", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "dp2a.lo.s32.s32 %o0, %i0, %i1, %i2;", + "prog": [ + 2476896, + 800 + ], + "rows": [ + 2477696, + 16384 + ] + }, + { + "name": "dp2a_hi", + "family": "bits", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "dp2a.hi.u32.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 2494080, + 800 + ], + "rows": [ + 2494880, + 16384 + ] + }, + { + "name": "shfl_idx", + "family": "warp", + "ins": "ul", + "outs": "u", + "cases": 1024, + "ptx": "shfl.sync.idx.b32 %o0, %i0, %i1, 0x1f, 0xffffffff;", + "prog": [ + 2511264, + 800 + ], + "rows": [ + 2512064, + 12288 + ] + }, + { + "name": "shfl_up", + "family": "warp", + "ins": "us", + "outs": "uu", + "cases": 1024, + "ptx": "and.b32 %t0, %i1, 31; shfl.sync.up.b32 %o0|%p0, %i0, %t0, 0, 0xffffffff; selp.u32 %o1, 1, 0, %p0;", + "prog": [ + 2524352, + 800 + ], + "rows": [ + 2525152, + 16384 + ] + }, + { + "name": "shfl_down", + "family": "warp", + "ins": "us", + "outs": "uu", + "cases": 1024, + "ptx": "and.b32 %t0, %i1, 31; shfl.sync.down.b32 %o0|%p0, %i0, %t0, 0x1f, 0xffffffff; selp.u32 %o1, 1, 0, %p0;", + "prog": [ + 2541536, + 800 + ], + "rows": [ + 2542336, + 16384 + ] + }, + { + "name": "shfl_bfly", + "family": "warp", + "ins": "ul", + "outs": "u", + "cases": 1024, + "ptx": "shfl.sync.bfly.b32 %o0, %i0, %i1, 0x1f, 0xffffffff;", + "prog": [ + 2558720, + 800 + ], + "rows": [ + 2559520, + 12288 + ] + }, + { + "name": "shfl_bfly_imm", + "family": "warp", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "shfl.sync.bfly.b32 %o0, %i0, 1, 0x1f, 0xffffffff;", + "prog": [ + 2571808, + 800 + ], + "rows": [ + 2572608, + 8192 + ] + }, + { + "name": "shfl_idx_seg", + "family": "warp", + "ins": "ul", + "outs": "u", + "cases": 1024, + "ptx": "shfl.sync.idx.b32 %o0, %i0, %i1, 0x1807, 0xffffffff;", + "prog": [ + 2580800, + 800 + ], + "rows": [ + 2581600, + 12288 + ] + }, + { + "name": "vote", + "family": "warp", + "ins": "p", + "outs": "uuuu", + "cases": 1024, + "ptx": "and.b32 %t0, %i0, 1; setp.ne.u32 %p0, %t0, 0; vote.sync.ballot.b32 %o0, %p0, 0xffffffff; vote.sync.any.pred %p1, %p0, 0xffffffff; selp.u32 %o1, 1, 0, %p1; vote.sync.all.pred %p2, %p0, 0xffffffff; selp.u32 %o2, 1, 0, %p2; vote.sync.uni.pred %p3, %p0, 0xffffffff; selp.u32 %o3, 1, 0, %p3;", + "prog": [ + 2593888, + 928 + ], + "rows": [ + 2594816, + 20480 + ] + }, + { + "name": "activemask", + "family": "warp", + "ins": "p", + "outs": "u", + "cases": 1024, + "ptx": "activemask.b32 %o0;", + "prog": [ + 2615296, + 800 + ], + "rows": [ + 2616096, + 8192 + ] + }, + { + "name": "match_any", + "family": "warp", + "ins": "p", + "outs": "u", + "cases": 1024, + "ptx": "and.b32 %t0, %i0, 3; match.any.sync.b32 %o0, %t0, 0xffffffff;", + "prog": [ + 2624288, + 800 + ], + "rows": [ + 2625088, + 8192 + ] + }, + { + "name": "redux_add", + "family": "warp", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "redux.sync.add.u32 %o0, %i0, 0xffffffff;", + "prog": [ + 2633280, + 800 + ], + "rows": [ + 2634080, + 8192 + ] + }, + { + "name": "redux_min_s32", + "family": "warp", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "redux.sync.min.s32 %o0, %i0, 0xffffffff;", + "prog": [ + 2642272, + 800 + ], + "rows": [ + 2643072, + 8192 + ] + }, + { + "name": "redux_or", + "family": "warp", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "redux.sync.or.b32 %o0, %i0, 0xffffffff;", + "prog": [ + 2651264, + 800 + ], + "rows": [ + 2652064, + 8192 + ] + }, + { + "name": "hmma_16816_f16", + "family": "tensor", + "ins": "HHHHHHHH", + "outs": "hh", + "cases": 1024, + "ptx": "mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%o0, %o1}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%i6, %i7};", + "prog": [ + 2660256, + 928 + ], + "rows": [ + 2661184, + 40960 + ] + }, + { + "name": "hmma_16816_f32", + "family": "tensor", + "ins": "HHHHHHFFFF", + "outs": "ffff", + "cases": 1024, + "ptx": "mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;", + "prog": [ + 2702144, + 928 + ], + "rows": [ + 2703072, + 57344 + ] + }, + { + "name": "hmma_1688_f16", + "family": "tensor", + "ins": "HHHHH", + "outs": "hh", + "cases": 1024, + "ptx": "mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%o0, %o1}, {%i0, %i1}, {%i2}, {%i3, %i4};", + "prog": [ + 2760416, + 800 + ], + "rows": [ + 2761216, + 28672 + ] + }, + { + "name": "hmma_16816_bf16", + "family": "tensor", + "ins": "BBBBBBFFFF", + "outs": "ffff", + "cases": 1024, + "ptx": "mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;", + "prog": [ + 2789888, + 928 + ], + "rows": [ + 2790816, + 57344 + ] + }, + { + "name": "hmma_1688_tf32", + "family": "tensor", + "ins": "FFFFFFFFFF", + "outs": "ffff", + "cases": 1024, + "ptx": "mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;", + "prog": [ + 2848160, + 928 + ], + "rows": [ + 2849088, + 57344 + ] + }, + { + "name": "imma_16832_s8", + "family": "tensor", + "ins": "UUUUUUUUUU", + "outs": "uuuu", + "cases": 1024, + "ptx": "mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%o0, %o1, %o2, %o3}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%i6, %i7, %i8, %i9};", + "prog": [ + 2906432, + 928 + ], + "rows": [ + 2907360, + 57344 + ] + }, + { + "name": "dadd", + "family": "f64", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %g0, {%i0, %i1}; mov.b64 %g1, {%i2, %i3}; add.rn.f64 %g2, %g0, %g1; mov.b64 {%o0, %o1}, %g2;", + "prog": [ + 2964704, + 800 + ], + "rows": [ + 2965504, + 24576 + ] + }, + { + "name": "dmul", + "family": "f64", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %g0, {%i0, %i1}; mov.b64 %g1, {%i2, %i3}; mul.rn.f64 %g2, %g0, %g1; mov.b64 {%o0, %o1}, %g2;", + "prog": [ + 2990080, + 800 + ], + "rows": [ + 2990880, + 24576 + ] + } + ] +} diff --git a/dm/gfx/tests/test_dmsh_ampere_orin.py b/dm/gfx/tests/test_dmsh_ampere_orin.py new file mode 100644 index 0000000000..be1857e6d7 --- /dev/null +++ b/dm/gfx/tests/test_dmsh_ampere_orin.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Ampere instructions through the whole chain against a real Ampere GPU: each op of ampere_orin/ (recorded on a Jetson Orin, sm_87, by +dm/gfx/tools/ampere_orin/capture.py) is the GPU compiler's machine code for one PTX operation, its 1,024 input rows and the outputs the GPU computed. Here the +program goes through the translator (dmsh_xlate, arch sm86, the build's debug and relaxed-precision options), spirv-cross with the chain's arguments and the +MSL rewrites dmsh_build.py's finish() makes of every shader, is compiled with the chain's Metal flags and run on this Mac's GPU on the same inputs, and every +output word is compared with the GPU's bit for bit. + +An op is expected to match except for what Metal's float arithmetic itself gives (compare.PLATFORM: NaN bits, denormals). KNOWN lists the ops that still +differ, each with its reason and where it is tracked; APPROX the ones that run Metal's own approximation of a function (bit-exact only by accident). The test +fails when an op outside those lists differs, and when a KNOWN op matches, so that its entry is taken out with the fix. + +Needs DMSH_TEST_XLATE (the translator) like the other translator tests, and an Apple-silicon Mac. AMPERE_ORIN_ONLY= runs some; +AMPERE_ORIN_REPORT= writes every op's causes as JSON.""" +import ast +import bisect +import concurrent.futures +import gzip +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +TOOLS = HERE.parent / 'tools' +FIXTURE = HERE / 'ampere_orin' +sys.path.insert(0, str(TOOLS)) +sys.path.insert(0, str(TOOLS / 'ampere_orin')) +import compare # noqa: E402 + +# Ops that still differ from the GPU beyond PLATFORM: name -> why. A fix takes its ops out of here. +KNOWN = { + # results that differ + 'fadd_rz': 'FADD .RZ / .RM / .RP: the rounding mode is dropped, Metal rounds to nearest', 'fadd_rm': '(as fadd_rz)', 'fadd_rp': '(as fadd_rz)', + 'fmul_rz': 'FMUL .RZ / .RM / .RP: the rounding mode is dropped', 'fmul_rm': '(as fmul_rz)', 'fmul_rp': '(as fmul_rz)', + 'ffma_rz': 'FFMA .RZ / .RM / .RP: the rounding mode is dropped', 'ffma_rm': '(as ffma_rz)', 'ffma_rp': '(as ffma_rz)', + 'i2f_s32_rz': 'I2FP.F32.S32.RZ rounds to nearest (PR #1800 has the fix)', + 'fmax_nan': 'FMNMX.NAN: a NaN operand gives the other operand, not NaN (PR #1800 has the fix)', 'fmin_nan': '(as fmax_nan)', + 'fsetp': 'FSETP.NE (ordered) becomes MSL !=, which is unordered: true with a NaN operand', 'fsetp_ftz': '(as fsetp)', + 'shfl_idx': 'SHFL: a lane in a register is not masked to its low 5 bits', 'shfl_bfly': '(as shfl_idx)', 'shfl_idx_seg': '(as shfl_idx)', + 'f2fp_pack_bf16x2': 'F2FP.BF16.PACK_AB packs f16, not bf16', 'f2fp_pack_f16x2_relu': 'F2FP.RELU: negative results are not clamped to 0', + 'fmax': 'FMNMX: Ampere orders -0 below +0 (max(+0, -0) is +0); Metal returns either', 'hmax2': '(as fmax, HMNMX2)', + 'hadd2_ftz': 'HADD2 / HMUL2 .FTZ: f16 denormals are not flushed (Metal keeps them)', 'hmul2_ftz': '(as hadd2_ftz)', + 'hmma_16816_f32': 'HMMA.16816.F32: the f32 accumulator does not add the products as the tensor core does (about 1 ULP)', + # the translator refuses these + 'hfma2': 'HFMA2.MMA (0x235) is not implemented', 'hfma2_ftz': '(as hfma2)', 'hfma2_sat': '(as hfma2)', 'hfma2_relu': '(as hfma2)', + 'mufu_rcp': 'HFMA2.MMA with an immediate (0x435) is not implemented', 'f2f_tf32': '(as mufu_rcp)', 'idiv_u32': '(as mufu_rcp)', + 'idiv_u32_by_7': '(as mufu_rcp)', 'irem_s32': '(as mufu_rcp)', 'fdiv_rn': '(as mufu_rcp)', 'f2i_s64': '(as mufu_rcp)', 'iadd64': '(as mufu_rcp)', + 'isub64': '(as mufu_rcp)', 'imul64': '(as mufu_rcp)', 'imul24': '(as mufu_rcp)', 'dadd': '(as mufu_rcp; then DADD)', 'dmul': '(as mufu_rcp; then DMUL)', + 'hmin2_nan': 'HMNMX2.NAN is not implemented', 'bmin2': 'HMNMX2.BF16_V2 is not implemented', 'bmax2_nan': '(as bmin2)', + 'bfma2': 'HFMA2.BF16_V2 is not implemented', 'hmma_16816_bf16': 'HMMA .BF16 is not implemented', 'hmma_1688_tf32': 'HMMA .TF32 is not implemented', + 'imma_16832_s8': 'IMMA is not implemented', 'dp4a_u32': 'IDP.4A is not implemented', 'dp4a_s32': '(as dp4a_u32)', 'dp4a_su': '(as dp4a_u32)', + 'dp2a_lo': 'IDP.2A is not implemented', 'dp2a_hi': '(as dp2a_lo)', 'i2i_s8_sat': 'I2I .SAT is not implemented', 'i2i_u16_sat': '(as i2i_s8_sat)', + 'isad': 'VABSDIFF is not implemented', 'iadd_sat': 'PLOP3 with .SIGN register sources (0x21f) is not implemented', 'isub_sat': '(as iadd_sat)', + 'prmt': 'PRMT modes other than the index form (.F4E .B4E .RC8 .ECL .ECR .RC16) are not implemented', + 'mufu_tanh': 'MUFU.TANH is not implemented', 'mufu_tanh_f16x2': '(as mufu_tanh)', + # the program calls a routine of the CUDA driver's own (CALL.ABS to a relocation the JIT resolves at load), which a game's shader never does + 'frcp_rn': 'not testable: CALL.ABS into the CUDA driver', 'fsqrt_rn': '(as frcp_rn)', +} +# Ops that run Metal's own approximation (MUFU, and the divisions built on it): compared and reported, never failed +APPROX = {'mufu_ex2', 'mufu_ex2_ftz', 'mufu_lg2', 'mufu_lg2_ftz', 'mufu_rcp_ftz', 'mufu_rsqrt', 'mufu_rsqrt_ftz', 'mufu_sqrt', 'mufu_sqrt_ftz', 'mufu_sin', + 'mufu_cos', 'mufu_ex2_f16x2', 'fdiv_full', 'fdiv_approx', + # HMMA with an f16 accumulator: one block of 8 rounded once (PR #1897) matches 99.97 %; the rest are 1-2 ULP where the sum nearly cancels + 'hmma_16816_f16', 'hmma_1688_f16'} + +MAC_METAL = ['xcrun', '-sdk', 'macosx', 'metal', '-c', '-O1', '-fpreserve-invariance', '-fwrapv'] # dmsh_build.py MAC_METAL + + +def load_fixture(): + doc = json.loads((FIXTURE / 'manifest.json').read_text()) + blob = gzip.decompress((FIXTURE / 'ampere_orin.u32.gz').read_bytes()) + import array + for o in doc['ops']: + o['program'] = blob[o['prog'][0]:o['prog'][0] + o['prog'][1]] + words = array.array('I') + words.frombytes(blob[o['rows'][0]:o['rows'][0] + o['rows'][1]]) + if sys.byteorder != 'little': + words.byteswap() + w = len(o['ins']) + len(o['outs']) + o['rows_in'] = [list(words[i:i + len(o['ins'])]) for i in range(0, len(words), w)] + o['rows_gpu'] = [list(words[i + len(o['ins']):i + w]) for i in range(0, len(words), w)] + return doc + + +def chain_rewrites(): + """the MSL rewrites dmsh_build.py's finish() makes of every shader, in its order: each `src = f(src)` at its top level or in a try there (the ones under a + stage's `if` are another stage's). dmsh_build.py runs as a script, so its functions and constants are taken from its source, as test_dmsh_hmma.py does""" + import dmsh_atomic + import dmsh_wide + path = TOOLS / 'dmsh_build.py' + tree = ast.parse(path.read_text()) + scope = {'re': re, 'bisect': bisect, 'os': os, 'json': json, 'dmsh_atomic': dmsh_atomic, 'dmsh_wide': dmsh_wide} + for n in tree.body: + pure = isinstance(n, ast.Assign) and all(isinstance(t, ast.Name) for t in n.targets) and not any( + isinstance(c, ast.Call) and getattr(c.func, 'attr', '') != 'compile' for c in ast.walk(n.value)) + if isinstance(n, ast.FunctionDef) or pure: + try: + exec(compile(ast.Module(body=[n], type_ignores=[]), str(path), 'exec'), scope) + except Exception: # a constant built from what only a build has (the game, the chain version) is not needed here + pass + finish = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'finish') + steps = [] + + def visit(body): + for st in body: + if isinstance(st, ast.Try): + visit(st.body) + elif (isinstance(st, ast.Assign) and len(st.targets) == 1 and getattr(st.targets[0], 'id', None) == 'src' and isinstance(st.value, ast.Call) + and len(st.value.args) == 1 and getattr(st.value.args[0], 'id', None) == 'src' and not st.value.keywords): + steps.append(ast.unparse(st.value.func)) + visit(finish.body) + fns = [] + for name in steps: + mod, _, attr = name.rpartition('.') + fns.append((name, getattr(scope[mod], attr) if mod else scope[name])) + return fns, scope['XC'] + + +def harness(msl, w): + """main0 seeds each thread's W words of the shader's shared array from buffer 30 and writes them back where it returns""" + i = msl.index('kernel void main0(') + sig = msl[i:msl.index(')', i)] + lid = re.search(r'uint (\w+) \[\[thread_index_in_threadgroup\]\]', sig) + lid3 = re.search(r'uint3 (\w+) \[\[thread_position_in_threadgroup\]\]', sig) + tg = re.search(r'uint3 (\w+) \[\[threadgroup_position_in_grid\]\]', sig) + add = ['device uint* dm_io [[buffer(30)]]'] + if lid: + lid_e = lid.group(1) + elif lid3: + lid_e = lid3.group(1) + '.x' + else: + add.append('uint dm_lid [[thread_index_in_threadgroup]]') + lid_e = 'dm_lid' + if tg: + tg_e = tg.group(1) + '.x' + else: + add.append('uint3 dm_tg [[threadgroup_position_in_grid]]') + tg_e = 'dm_tg.x' + rest = msl[i + len('kernel void main0('):] + msl = msl[:i] + 'kernel void main0(' + ', '.join(add) + (', ' if not rest.lstrip().startswith(')') else '') + rest + start = msl.index('{', msl.index(')', msl.index('kernel void main0('))) + 1 + end = msl.rindex('}') + body = msl[start:end] + shm = re.search(r'threadgroup spvUnsafeArray (\w+);|threadgroup uint (\w+)\[(\d+)\];', body) + if not shm: + raise ValueError('no shared array in main0') + arr, words = shm.group(2) or shm.group(3), int(shm.group(1) or shm.group(4)) + if words < 32 * w: + raise ValueError(f'shared array of {words} words, {32 * w} needed') + base = f'(({tg_e}) * 32u + ({lid_e})) * {w}u' + dump = f'for (uint dm_k = 0; dm_k < {w}u; dm_k++) dm_io[{base} + dm_k] = {arr}[({lid_e}) * {w}u + dm_k];' + body = (body[:shm.end()] + f'\n for (uint dm_k = 0; dm_k < {w}u; dm_k++) {arr}[({lid_e}) * {w}u + dm_k] = dm_io[{base} + dm_k];\n' + + body[shm.end():]) + body = re.sub(r'\breturn;', '{ ' + dump + ' return; }', body).rstrip() + '\n ' + dump + '\n' + return msl[:start] + body + msl[end:] + + +RUNNER = r'''#import +#import +// run : main0 over groups of 32 threads, io at buffer 30 (the other slots a zero buffer), written back over io +int main(int argc, char **argv) { @autoreleasepool { + id d = MTLCreateSystemDefaultDevice(); NSError *e = nil; + if (!d) { fprintf(stderr, "no Metal device\n"); return 1; } + id lib = [d newLibraryWithURL:[NSURL fileURLWithPath:@(argv[1])] error:&e]; if (!lib) { fprintf(stderr, "library: %s\n", e.description.UTF8String); return 1; } + id f = [lib newFunctionWithName:@"main0"]; if (!f) { fprintf(stderr, "no main0\n"); return 1; } + id ps = [d newComputePipelineStateWithFunction:f error:&e]; if (!ps) { fprintf(stderr, "pipeline: %s\n", e.description.UTF8String); return 1; } + NSData *in = [NSData dataWithContentsOfFile:@(argv[2])]; if (!in) { fprintf(stderr, "no input\n"); return 1; } + id b = [d newBufferWithBytes:in.bytes length:in.length options:MTLResourceStorageModeShared]; + id zero = [d newBufferWithLength:65536 options:MTLResourceStorageModeShared]; + id q = [d newCommandQueue]; id cb = [q commandBuffer]; id ce = [cb computeCommandEncoder]; + if (!q || !cb || !ce || !b || !zero) { fprintf(stderr, "Metal objects\n"); return 1; } + [ce setComputePipelineState:ps]; + for (int i = 0; i < 30; i++) [ce setBuffer:zero offset:0 atIndex:i]; + [ce setBuffer:b offset:0 atIndex:30]; + [ce dispatchThreadgroups:MTLSizeMake(atoi(argv[3]), 1, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)]; + [ce endEncoding]; [cb commit]; [cb waitUntilCompleted]; + if (cb.error) { fprintf(stderr, "gpu: %s\n", cb.error.description.UTF8String); return 1; } + [[NSData dataWithBytes:b.contents length:in.length] writeToFile:@(argv[2]) atomically:YES]; return 0; } } +''' + + +def run_op(o, work, runner, steps, xc, env): + """('ok', metal rows) or (stage that failed, its message)""" + import array + import dmsh_atomic + import dmsh_geometry + d = work / o['name'] + d.mkdir() + (d / 'k.bin').write_bytes(o['program']) + w = len(o['ins']) + len(o['outs']) + job = [dict(key='k', stage='cs', arch='sm86', code=str(d / 'k.bin'), tex={}, img={}, attr={}, fout={}, local=[32, 1, 1], debug=True, reduced=True, + nvn2_cs=[0, 0, 3, 0x40, 0], shared=32 * w * 4)] + (d / 'jobs.json').write_text(json.dumps(job)) + r = subprocess.run([os.environ['DMSH_TEST_XLATE'], str(d / 'jobs.json'), str(d)], capture_output=True, text=True, env=env) + if r.returncode or not (d / 'k.spv').exists(): + return 'translator', next((ln for ln in (r.stdout + r.stderr).splitlines() if '[RED]' in ln), (r.stdout + r.stderr).strip()[-300:]) + dmsh_geometry.flatten_sets(d / 'k.spv') + r = subprocess.run([str(TOOLS / 'spirv_cross_bin/spirv-cross'), *dmsh_atomic.cross_args(xc, str(d / 'k.spv')), str(d / 'k.spv')], capture_output=True, text=True) + if r.returncode: + return 'spirv-cross', r.stderr.strip()[-300:] + src = r.stdout + try: + for _, fn in steps: + src = fn(src) + src = harness(src, w) + except Exception as e: + return 'MSL rewrite', f'{type(e).__name__}: {e}' + (d / 'k.metal').write_text(src) + r = subprocess.run(MAC_METAL + [str(d / 'k.metal'), '-o', str(d / 'k.air')], capture_output=True, text=True) + if r.returncode: + return 'Metal compile', r.stderr.strip()[-300:] + subprocess.run(['xcrun', '-sdk', 'macosx', 'metallib', str(d / 'k.air'), '-o', str(d / 'k.metallib')], check=True, capture_output=True) + io = array.array('I', [0xcdcdcdcd]) * (len(o['rows_in']) * w) + for c, row in enumerate(o['rows_in']): + io[c * w:c * w + len(row)] = array.array('I', row) + (d / 'io.bin').write_bytes(io.tobytes()) + r = subprocess.run([str(runner), str(d / 'k.metallib'), str(d / 'io.bin'), str(len(o['rows_in']) // 32)], capture_output=True, text=True, timeout=120) + if r.returncode: + return 'Metal run', r.stderr.strip()[-300:] + out = array.array('I') + out.frombytes((d / 'io.bin').read_bytes()) + return 'ok', [list(out[c * w + len(o['ins']):(c + 1) * w]) for c in range(len(o['rows_in']))] + + +@unittest.skipUnless(os.environ.get('DMSH_TEST_XLATE'), 'set DMSH_TEST_XLATE to the offline translator') +@unittest.skipUnless(sys.platform == 'darwin' and shutil.which('xcrun'), 'needs a Mac with Xcode') +class AgainstAmpere(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.doc = load_fixture() + only = set(filter(None, os.environ.get('AMPERE_ORIN_ONLY', '').split(','))) + ops = [o for o in cls.doc['ops'] if not only or o['name'] in only or o['family'] in only] + steps, xc = chain_rewrites() + cls.tmp = tempfile.TemporaryDirectory(prefix='ampere_orin_') + work = Path(cls.tmp.name) + (work / 'run.m').write_text(RUNNER) + subprocess.run(['clang', '-fobjc-arc', '-O1', str(work / 'run.m'), '-framework', 'Metal', '-framework', 'Foundation', '-o', str(work / 'run')], check=True) + env = dict(os.environ) + with concurrent.futures.ThreadPoolExecutor(max(2, (os.cpu_count() or 4) - 2)) as ex: + futs = {o['name']: ex.submit(run_op, o, work, work / 'run', steps, xc, env) for o in ops} + cls.results = {} + for o in ops: + stage, got = futs[o['name']].result() + if stage == 'ok': + causes, lanes, worst, examples = compare.classify(o, o['rows_in'], o['rows_gpu'], got) + cls.results[o['name']] = dict(stage='ok', lanes=lanes, causes=dict(causes), worst_ulp=worst, examples=examples) + else: + cls.results[o['name']] = dict(stage=stage, message=got) + cls.ops = ops + if os.environ.get('AMPERE_ORIN_REPORT'): + Path(os.environ['AMPERE_ORIN_REPORT']).write_text(json.dumps({'device': cls.doc['device'], 'rewrites': [n for n, _ in steps], 'ops': cls.results}, indent=1)) + cls.print_summary() + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + @classmethod + def print_summary(cls): + lines = [f"\nAmpere ops against the {cls.doc['device'].get('gpu', 'GPU')} (sm_{cls.doc['device'].get('sm', '?')}, CUDA driver " + f"{cls.doc['device'].get('cuda_driver', '?')}): lanes that differ, by cause"] + for o in cls.ops: + r = cls.results[o['name']] + tag = 'known' if o['name'] in KNOWN else 'approx' if o['name'] in APPROX else '' + if r['stage'] != 'ok': + lines.append(f" {o['name']:22s} {tag:6s} {r['stage']} failed: {r['message'][:120]}") + continue + diff = sum(r['causes'].values()) + cs = ', '.join(f'{k} {v}' for k, v in sorted(r['causes'].items(), key=lambda kv: -kv[1])) + lines.append(f" {o['name']:22s} {tag:6s} {diff:5d} of {r['lanes']:6d}" + (f' ({cs})' if cs else '') + (f" max {r['worst_ulp']} ULP" if r['worst_ulp'] else '')) + print('\n'.join(lines), file=sys.stderr) + + def test_matches_the_gpu(self): + for o in self.ops: + n, r = o['name'], self.results[o['name']] + with self.subTest(op=n): + beyond = {k: v for k, v in r.get('causes', {}).items() if k not in compare.PLATFORM} + if n in APPROX: + self.assertEqual(r['stage'], 'ok', r.get('message')) + elif n in KNOWN: + self.assertTrue(r['stage'] != 'ok' or beyond, f'{n} now matches the GPU: take it out of KNOWN ({KNOWN[n]})') + else: + self.assertEqual(r['stage'], 'ok', f"{n}: {r['stage']} failed: {r.get('message')}") + self.assertFalse(beyond, f"{n} differs from the GPU: {beyond}; for example {r['examples'][:2]}") + + def test_every_listed_op_is_recorded(self): # a KNOWN or APPROX entry for an op the fixture lacks is a typo + names = {o['name'] for o in self.doc['ops']} + self.assertFalse((set(KNOWN) | APPROX) - names) + + +if __name__ == '__main__': + unittest.main() diff --git a/dm/gfx/tools/ampere_orin/batch.py b/dm/gfx/tools/ampere_orin/batch.py new file mode 100644 index 0000000000..0c789f8cf7 --- /dev/null +++ b/dm/gfx/tools/ampere_orin/batch.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""On the Jetson: for every op in manifest.json, JIT its PTX to a cubin, disassemble it with nvdisasm, and run it on the op's inputs. + + batch.py manifest.json (run from the folder holding the ops' .ptx and .in.bin files) + +Writes .cubin, .sass and .orin.bin beside them, and .err when a step fails. Stdlib only, through cu.py.""" +import json +import os +import subprocess +import sys +import traceback + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import cu # noqa: E402 +import prog # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def main(): + import ctypes as C + cu.cu.cuGetErrorString.argtypes = [C.c_int, C.POINTER(C.c_char_p)] + cu.cu.cuLinkAddData_v2.argtypes = [C.c_void_p, C.c_int, C.c_void_p, C.c_size_t, C.c_char_p, C.c_uint, C.c_void_p, C.c_void_p] + cu.cu.cuMemcpyHtoD_v2.argtypes = [C.c_uint64, C.c_char_p, C.c_size_t] + cu.cu.cuMemcpyDtoH_v2.argtypes = [C.c_void_p, C.c_uint64, C.c_size_t] + cu.cu.cuMemsetD8_v2.argtypes = [C.c_uint64, C.c_ubyte, C.c_size_t] + cu.cu.cuMemFree_v2.argtypes = [C.c_uint64] + cu.cu.cuLaunchKernel.argtypes = [C.c_void_p] + [C.c_uint] * 7 + [C.c_void_p, C.c_void_p, C.c_void_p] + cu.init() + ops = json.load(open(sys.argv[1])) + ok = 0 + name, drv = C.create_string_buffer(256), C.c_int() + cu.cu.cuDeviceGetName(name, 256, 0) + cu.cu.cuDriverGetVersion(C.byref(drv)) + maj, mnr = C.c_int(), C.c_int() + cu.cu.cuDeviceGetAttribute(C.byref(maj), 75, 0) # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR + cu.cu.cuDeviceGetAttribute(C.byref(mnr), 76, 0) # ..._MINOR + nvd = subprocess.run([os.path.join(HERE, 'nvdisasm'), '--version'], capture_output=True, text=True).stdout.strip().splitlines()[-1:] + json.dump({'gpu': name.value.decode(), 'sm': f'{maj.value}{mnr.value}', 'cuda_driver': f'{drv.value // 1000}.{drv.value % 1000 // 10}', + 'nvdisasm': nvd[0] if nvd else ''}, open('device.info', 'w')) + for op in ops: + n = op['name'] + for ext in ('.err', '.orin.bin', '.prog.bin', '.prog.sass'): + if os.path.exists(n + ext): + os.remove(n + ext) + try: + cubin = cu.ptx2cubin(open(n + '.ptx', 'rb').read()) + open(n + '.cubin', 'wb').write(cubin) + sass = subprocess.run([os.path.join(HERE, 'nvdisasm'), '-hex', '-c', n + '.cubin'], capture_output=True, text=True, check=True).stdout + open(n + '.sass', 'w').write(sass) + out = cu.run(cubin, 'k', op['groups'], 32, 0, [open(n + '.in.bin', 'rb').read()], op['groups'] * 32 * op['nout'] * 4) + open(n + '.orin.bin', 'wb').write(out) + code, rb, notes = prog.rewrite(cubin, sass, op['nin'], op['nout']) # the Metal side's program, checked by the disassembler here + open(n + '.prog.bin', 'wb').write(code) + dis = subprocess.run([os.path.join(HERE, 'nvdisasm'), '--binary', 'SM87', '-hex', n + '.prog.bin'], capture_output=True, text=True, check=True).stdout + open(n + '.prog.sass', 'w').write(f'// base register R{rb}\n' + ''.join(f'// {x}\n' for x in notes) + dis) + ok += 1 + except BaseException as e: # cu.ck raises SystemExit + open(n + '.err', 'w').write(f'{type(e).__name__}: {e}\n{traceback.format_exc()}') + print(f'{n}: {e}', file=sys.stderr) + if 'kernel' in str(e) or 'cuLaunchKernel' in str(e): # a fault poisons the context: start a fresh one + cu.init() + print(f'{ok}/{len(ops)} ops ran') + + +if __name__ == '__main__': + main() diff --git a/dm/gfx/tools/ampere_orin/capture.py b/dm/gfx/tools/ampere_orin/capture.py new file mode 100644 index 0000000000..3307775e72 --- /dev/null +++ b/dm/gfx/tools/ampere_orin/capture.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Records the fixture of dm/gfx/tests/test_dmsh_ampere_orin.py on a Linux machine with an Ampere GPU and its NVIDIA driver (a Jetson Orin, sm_87, or a GeForce +RTX 30, sm_86). Needs no CUDA toolkit there, only Python 3 and the driver's libcuda: the driver compiles each op's PTX itself. + +For each op in ops.py: a PTX kernel that loads its inputs from global memory, does the op and stores its outputs. On the device (over ssh) the driver compiles +and runs it on the inputs made here, and prog.py rewrites the same machine code for the translator (LDG / STG become LDS / STS, the CUDA parameter reads and +the addressing they feed become NOPs), which the device's nvdisasm checks. The fixture keeps, per op, that program, its disassembly, the inputs and the GPU's +outputs. + + capture.py --host user@machine [--arch sm_87] [--cases 1024] [--only names,families] [--out dm/gfx/tests/ampere_orin] + +The device needs nvdisasm in --device-dir (NVIDIA's cuda_nvdisasm redistributable for linux-aarch64 or linux-x86_64; no install). PTX is written here, so no +compiler is needed on either side. Needs numpy here (.github/requirements/tools.txt).""" +import argparse +import gzip +import json +import re +import subprocess +import sys +from pathlib import Path + +import numpy as np # pyright: ignore[reportMissingImports] # numpy is in .github/requirements/tools.txt, not in the lint environment + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from ops import OPS # noqa: E402 + +FIXTURE = HERE.parents[1] / 'tests' / 'ampere_orin' + +F32_SPECIAL = [0x00000000, 0x80000000, 0x00000001, 0x807fffff, 0x00800000, 0x00000100, 0x3f800000, 0xbf800000, 0x3f000000, 0x7f7fffff, 0xff7fffff, + 0x7f800000, 0xff800000, 0x7fc00000, 0xffc00000, 0x7f800001, 0x7fffffff, 0x4b800000, 0x4b800001, 0x3effffff, 0x3f7fffff, 0x40490fdb, + 0x4f000000, 0xcf000000, 0x4f800000, 0x5f000000, 0x3fc00000, 0x40200000, 0xbfc00000, 0x33800000, 0x38800000, 0x477fe000, 0x477ff000, + 0x33000000, 0x3f800001, 0x4effffff, 0xceffffff, 0x3f7ffffe, 0x4b000000, 0xcb000000, 0x4b7fffff, 0x3ea2f983, 0x40c90fdb, 0x42b17218] +F16_SPECIAL = [0x0000, 0x8000, 0x0001, 0x03ff, 0x0400, 0x3c00, 0xbc00, 0x3800, 0x7bff, 0xfbff, 0x7c00, 0xfc00, 0x7e00, 0x7c01, 0x7fff, 0xfe00, + 0x3555, 0x3bff, 0x3c01, 0x0200, 0x8001, 0x6400, 0x4248, 0x5640] +BF16_SPECIAL = [0x0000, 0x8000, 0x0001, 0x007f, 0x0080, 0x3f80, 0xbf80, 0x3f00, 0x7f7f, 0xff7f, 0x7f80, 0xff80, 0x7fc0, 0x7f81, 0x7fff, 0x4049] +U32_SPECIAL = [0, 1, 2, 3, 7, 0xffffffff, 0xfffffffe, 0x80000000, 0x7fffffff, 0x7ffffffe, 0x80000001, 0xff, 0x100, 0xffff, 0x10000, 0x55555555, + 0xaaaaaaaa, 31, 32, 33, 63, 64, 0xffffffe1, 0x00ff00ff, 0x12345678, 0x80808080, 0x7f7f7f7f] + + +def _halves(kind, m, rng, tame): + def pack(v): + return (v.astype(np.float16).view(np.uint16) if kind == 'h' else (v.astype(np.float32).view(np.uint32) >> 16).astype(np.uint16)).astype(np.uint32) + if tame: + return pack(rng.uniform(-1, 1, m)) + p = rng.random(m) + a = rng.choice(np.array(F16_SPECIAL if kind == 'h' else BF16_SPECIAL, np.uint32), m) + b = rng.integers(0, 2 ** 16, m).astype(np.uint32) + c = pack(rng.uniform(-4, 4, m)) + d = pack(rng.choice([-1.0, 1.0], m) * np.exp2(rng.uniform(-14, 15, m))) + return np.select([p < .3, p < .45, p < .75], [a, b, c], d).astype(np.uint32) + + +def gen_kind(kind, n, rng): + """n input words of one kind (ops.py): special values, random bits and ordinary values mixed; an upper-case kind gives ordinary values only""" + tame, k = kind.isupper(), kind.lower() + pick = rng.random(n) + rand = rng.integers(0, 2 ** 32, n, dtype=np.uint64).astype(np.uint32) + if k == 'f': + if tame: + return rng.uniform(-2, 2, n).astype(np.float32).view(np.uint32) + a = rng.choice(np.array(F32_SPECIAL, np.uint32), n) + c = rng.uniform(-8, 8, n).astype(np.float32).view(np.uint32) + d = (rng.choice([-1.0, 1.0], n) * np.exp2(rng.uniform(-30, 30, n))).astype(np.float32).view(np.uint32) + e = (rng.integers(-4, 5, n) * 0.5).astype(np.float32).view(np.uint32) # halves and whole numbers: the ties of the roundings + return np.select([pick < .25, pick < .4, pick < .65, pick < .85], [a, rand, c, d], e).astype(np.uint32) + if k in 'hb': + return (_halves(k, n, rng, tame) | _halves(k, n, rng, tame) << 16).astype(np.uint32) + if k == 'u': + if tame: + return rand + a = rng.choice(np.array(U32_SPECIAL, np.uint32), n) + c = rng.integers(-1000, 1000, n).astype(np.int32).view(np.uint32) + return np.select([pick < .3, pick < .7], [a, rand], c).astype(np.uint32) + if k == 's': + a = rng.integers(0, 41, n).astype(np.uint32) + c = rng.choice(np.array([0, 1, 31, 32, 33, 63, 64, 255, 256, 0xffffffff, 0x80000000, 0xffffffe0], np.uint32), n) + return np.select([pick < .7, pick < .85], [a, rand], c).astype(np.uint32) + if k == 'l': + return np.where(pick < .8, rng.integers(0, 32, n).astype(np.uint32), rand).astype(np.uint32) + if k == 'p': # whole warps of one value now and then, so all / any / uni see both answers + same = np.repeat(rng.random(n // 32) < .3, 32) + return np.where(same, np.repeat(rng.integers(0, 2 ** 32, n // 32, dtype=np.uint64).astype(np.uint32), 32), rand).astype(np.uint32) + raise ValueError(kind) + + +PTX_HEAD = '''.version 8.5 +.target {arch} +.address_size 64 +.visible .entry k(.param .u64 pin, .param .u64 pout) +{ + .reg .b32 %i<{nin}>, %o<{nout}>, %t<8>, %r<8>, %x<8>; + .reg .b16 %y<8>, %w<8>; + .reg .f32 %f<8>; + .reg .b64 %d<8>, %a<4>; + .reg .f64 %g<4>; + .reg .pred %p<4>; + mov.u32 %r0, %tid.x; mov.u32 %r1, %ctaid.x; shl.b32 %r1, %r1, 5; add.u32 %r2, %r1, %r0; + ld.param.u64 %a0, [pin]; cvta.to.global.u64 %a0, %a0; mul.wide.u32 %a1, %r2, {inb}; add.u64 %a0, %a0, %a1; +''' + + +def ptx_for(o, arch): + """the op's kernel: volatile loads and stores, so the compiler keeps one 32-bit LDG / STG a word, which prog.py replaces one for one""" + nin, nout = len(o['ins']), len(o['outs']) + s = PTX_HEAD.replace('{arch}', arch).replace('{nin}', str(max(nin, 1))).replace('{nout}', str(nout)).replace('{inb}', str(nin * 4)) + s += ''.join(f' ld.volatile.global.b32 %i{k}, [%a0+{k * 4}];\n' for k in range(nin)) + s += ' ' + o['body'] + '\n' + s += f' ld.param.u64 %a2, [pout]; cvta.to.global.u64 %a2, %a2; mul.wide.u32 %a3, %r2, {nout * 4}; add.u64 %a2, %a2, %a3;\n' + s += ''.join(f' st.volatile.global.b32 [%a2+{k * 4}], %o{k};\n' for k in range(nout)) + return s + ' ret;\n}\n' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--host', required=True, help='ssh destination of the machine with the Ampere GPU') + ap.add_argument('--arch', default='sm_87', help='the GPU\'s PTX target: sm_87 (Jetson Orin), sm_86 (GeForce RTX 30)') + ap.add_argument('--cases', type=int, default=1024, help='cases an op, a multiple of 32 (one warp a group, as on the Metal side)') + ap.add_argument('--only', default='', help='op names or families, comma-separated') + ap.add_argument('--out', default=str(FIXTURE)) + ap.add_argument('--device-dir', default='ampere_orin', help='folder on the device (relative to its home) holding nvdisasm; the run uses /run') + ap.add_argument('--work', default='', help='local folder for the device\'s files (default: a temporary one)') + args = ap.parse_args() + if args.cases % 32: + sys.exit('--cases must be a multiple of 32') + want = set(args.only.split(',')) if args.only else None + ops = [o for o in OPS if not want or o['name'] in want or o['family'] in want] + import tempfile + with tempfile.TemporaryDirectory(prefix='ampere_orin_') as tmp: + work = Path(args.work or tmp) + work.mkdir(parents=True, exist_ok=True) + man = [] + for idx, o in enumerate(ops): + rng = np.random.default_rng(0x5eed + idx * 7919 + sum(map(ord, o['name']))) + data = np.stack([gen_kind(k, args.cases, rng) for k in o['ins']], 1) + (work / f"{o['name']}.ptx").write_text(ptx_for(o, args.arch)) + data.astype('/dev/null'], stdout=subprocess.PIPE) + subprocess.run(['tar', 'xf', '-', '-C', str(work)], stdin=tar.stdout, check=False) + tar.wait() + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + write_fixture(out, work, man, ops, args.arch) + + +def trimmed_sass(text): + """a program's disassembly without the hex column and the NOP padding after its last EXIT""" + lines = [ln for ln in text.splitlines() if ln.startswith('//') or '/*0' in ln] + lines = [re.sub(r'\s*/\* 0x[0-9a-f]+ \*/', '', ln).rstrip() for ln in lines] + while lines and lines[-1].rstrip(' ;').endswith('NOP'): + lines.pop() + out, run = [], [] + for ln in lines + ['']: + if ln.rstrip(' ;').endswith('NOP'): + run.append(ln) + continue + out += run if len(run) < 3 else [run[0], f' ... ({len(run) - 2} more NOPs)', run[-1]] + run = [] + out.append(ln) + return out[:-1] + + +def write_fixture(out, work, man, ops, arch): + """out/manifest.json (the device; per op its kinds, PTX body, case count and where its words lie), out/ampere_orin.u32.gz (per op: the program, then its + cases, a row of input words and the GPU's output words each) and out/ampere_orin.sass (each program's disassembly). Ops this run did not record keep their + old entries""" + info = json.loads((work / 'device.info').read_text()) if (work / 'device.info').exists() else {} + old_doc = json.loads((out / 'manifest.json').read_text()) if (out / 'manifest.json').exists() else {'ops': []} + old_blob = gzip.decompress((out / 'ampere_orin.u32.gz').read_bytes()) if (out / 'ampere_orin.u32.gz').exists() else b'' + old_sass = {} + if (out / 'ampere_orin.sass').exists(): + for part in (out / 'ampere_orin.sass').read_text().split('\n// == ')[1:]: + name, _, body = part.partition('\n') + old_sass[name.strip()] = body.rstrip('\n').splitlines() + entries = {} + for o in old_doc.get('ops', []): + if 'prog' not in o: + continue + prog = old_blob[o['prog'][0]:o['prog'][0] + o['prog'][1]] + rows = old_blob[o['rows'][0]:o['rows'][0] + o['rows'][1]] + entries[o['name']] = ({k: v for k, v in o.items() if k not in ('prog', 'rows')}, prog, rows, old_sass.get(o['name'], [])) + for o in man: + n = o['name'] + if not (work / f'{n}.orin.bin').exists() or not (work / f'{n}.prog.bin').exists(): + err = (work / f'{n}.err').read_text().splitlines()[0] if (work / f'{n}.err').exists() else 'no result' + print(f'{n}: left out ({err})', file=sys.stderr) + continue + ins = np.fromfile(work / f'{n}.in.bin', '> 16, kind)] + return [(word, kind)] + + +def denorm_input(row, kinds): + for v, k in zip(row, kinds): + k = k.lower() + if k == 'f' and is_denorm(v, 'f'): + return True + return False + + +def _f32(v): + return struct.unpack(' 0 else 0xff800000 + + +def rn_reference(name, row): + """the round-to-nearest result of a directed-rounding op on one case, or None. Doubles hold a sum or product of two floats exactly (and an FMA's well enough + for this label), so rounding the double once to float is the nearest float""" + mode = name.rsplit('_', 1)[-1] + if mode not in ('rz', 'rm', 'rp'): + return None + if name.startswith(('fadd_', 'fmul_', 'ffma_')): + a, b = _f32(row[0]), _f32(row[1]) + if name.startswith('fadd_'): + r = a + b + elif name.startswith('fmul_'): + r = a * b + else: + r = a * b + _f32(row[2]) + return _bits(r) if r == r else None + if name.startswith('i2f_s32_'): + return _bits(float(row[0] - (1 << 32) if row[0] >> 31 else row[0])) + return None + + +def classify(op, rows, orin, metal): + """(Counter of causes over differing lanes, number of lanes, the largest ULP distance among 'other' float lanes, up to four differing cases as dicts)""" + causes, n, worst, examples = Counter(), 0, 0, [] + kinds_in, kinds_out = op['ins'], op['outs'] + for r, (row, o_row, m_row) in enumerate(zip(rows, orin, metal)): + din = None + rn = None + row_causes = [] + for k, (ow, mw) in enumerate(zip(o_row, m_row)): + kind = kinds_out[k] + for (a, lk), (b, _) in zip(lanes(ow, kind), lanes(mw, kind)): + n += 1 + if a == b: + continue + if lk == 'u': + if din is None: + din = denorm_input(row, kinds_in) + cause = 'denorm-in' if din else 'other' + elif is_nan(a, lk) and is_nan(b, lk): + cause = 'nan-bits' + else: + if din is None: + din = denorm_input(row, kinds_in) + if din: + cause = 'denorm-in' + elif is_nan(a, lk) or is_nan(b, lk): + cause = 'nan-one' + elif lk == 'f' and (is_denorm(a, lk) or is_denorm(b, lk) or {a & 0x7fffffff, b & 0x7fffffff} == {0, 0x00800000}): + cause = 'denorm-out' + elif is_zero(a, lk) and is_zero(b, lk): + cause = 'zero-sign' + else: + if rn is None and len(kinds_out) == 1: + rn = rn_reference(op['name'], row) or -1 + if rn == b: + cause = 'rn-result' + else: + cause = 'other' + worst = max(worst, ulps(a, b, lk)) + causes[cause] += 1 + row_causes.append(cause) + if row_causes and len(examples) < 4 and any(c not in PLATFORM for c in row_causes): + examples.append({'lane': r % 32, 'in': [f'{v:#010x}' for v in row], 'gpu': [f'{v:#010x}' for v in o_row], + 'metal': [f'{v:#010x}' for v in m_row], 'why': sorted(set(row_causes))}) + return causes, n, worst, examples diff --git a/dm/gfx/tools/ampere_orin/cu.py b/dm/gfx/tools/ampere_orin/cu.py new file mode 100644 index 0000000000..beaa130572 --- /dev/null +++ b/dm/gfx/tools/ampere_orin/cu.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Runs a kernel on the Jetson through libcuda alone (no toolkit, no numpy): the driver JIT-compiles PTX for this GPU, or loads a cubin as given. + + cu.py ptx2cubin k.ptx k.cubin JIT the PTX for this GPU and save the cubin the linker produced + cu.py run k.(ptx|cubin) kernel grid block shared in.bin out_bytes out.bin [more in.bin ...] + launches kernel(in0, [in1, ...,] out) with grid x block threads and `shared` bytes of dynamic shared memory + +Every buffer is a plain device allocation; the kernel gets their addresses in argument order, the output last.""" +import ctypes as C +import sys + +cu = C.CDLL('libcuda.so.1') + + +def ck(r, what): + if r: + s = C.c_char_p() + cu.cuGetErrorString(r, C.byref(s)) + raise SystemExit(f'{what}: {r} {s.value.decode() if s.value else ""}') + + +def init(): + ck(cu.cuInit(0), 'cuInit') + dev = C.c_int() + ck(cu.cuDeviceGet(C.byref(dev), 0), 'cuDeviceGet') + ctx = C.c_void_p() + ck(cu.cuDevicePrimaryCtxRetain(C.byref(ctx), dev), 'cuDevicePrimaryCtxRetain') + ck(cu.cuCtxSetCurrent(ctx), 'cuCtxSetCurrent') + return dev + + +CU_JIT_INFO_LOG_BUFFER, CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES = 3, 4 +CU_JIT_ERROR_LOG_BUFFER, CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES = 5, 6 +CU_JIT_INPUT_PTX = 1 + + +def ptx2cubin(ptx: bytes) -> bytes: + log = C.create_string_buffer(1 << 16) + opts = (C.c_int * 2)(CU_JIT_ERROR_LOG_BUFFER, CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES) + vals = (C.c_void_p * 2)(C.cast(log, C.c_void_p), C.c_void_p(len(log))) + st = C.c_void_p() + ck(cu.cuLinkCreate_v2(2, opts, vals, C.byref(st)), 'cuLinkCreate') + data = C.create_string_buffer(ptx + b'\0') + r = cu.cuLinkAddData_v2(st, CU_JIT_INPUT_PTX, data, C.c_size_t(len(ptx) + 1), b'k.ptx', 0, None, None) + if r: + sys.stderr.write(log.value.decode(errors='replace')) + ck(r, 'cuLinkAddData') + out, size = C.c_void_p(), C.c_size_t() + ck(cu.cuLinkComplete(st, C.byref(out), C.byref(size)), 'cuLinkComplete') + cubin = C.string_at(out, size.value) + cu.cuLinkDestroy(st) + return cubin + + +def run(image: bytes, kernel: str, grid: int, block: int, shared: int, inputs, out_bytes: int) -> bytes: + mod = C.c_void_p() + img = C.create_string_buffer(image + (b'\0' if image[:4] != b'\x7fELF' else b'')) + ck(cu.cuModuleLoadData(C.byref(mod), img), 'cuModuleLoadData') + fn = C.c_void_p() + ck(cu.cuModuleGetFunction(C.byref(fn), mod, kernel.encode()), 'cuModuleGetFunction') + if shared > 48 * 1024: + ck(cu.cuFuncSetAttribute(fn, 8, shared), 'cuFuncSetAttribute(max dynamic shared)') # CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES + ptrs = [] + for data in inputs: + p = C.c_uint64() + ck(cu.cuMemAlloc_v2(C.byref(p), C.c_size_t(max(len(data), 4))), 'cuMemAlloc') + ck(cu.cuMemcpyHtoD_v2(p, data, C.c_size_t(len(data))), 'cuMemcpyHtoD') + ptrs.append(p) + o = C.c_uint64() + ck(cu.cuMemAlloc_v2(C.byref(o), C.c_size_t(out_bytes)), 'cuMemAlloc') + ck(cu.cuMemsetD8_v2(o, 0xcd, C.c_size_t(out_bytes)), 'cuMemsetD8') # a fill no result is likely to equal, so an unwritten word shows + ptrs.append(o) + args = (C.c_void_p * len(ptrs))(*[C.cast(C.byref(p), C.c_void_p) for p in ptrs]) + ck(cu.cuLaunchKernel(fn, grid, 1, 1, block, 1, 1, shared, None, args, None), 'cuLaunchKernel') + ck(cu.cuCtxSynchronize(), 'kernel') + buf = C.create_string_buffer(out_bytes) + ck(cu.cuMemcpyDtoH_v2(buf, o, C.c_size_t(out_bytes)), 'cuMemcpyDtoH') + for p in ptrs: + cu.cuMemFree_v2(p) + cu.cuModuleUnload(mod) + return buf.raw + + +if __name__ == '__main__': + cu.cuGetErrorString.argtypes = [C.c_int, C.POINTER(C.c_char_p)] + cu.cuLinkAddData_v2.argtypes = [C.c_void_p, C.c_int, C.c_void_p, C.c_size_t, C.c_char_p, C.c_uint, C.c_void_p, C.c_void_p] + cu.cuMemcpyHtoD_v2.argtypes = [C.c_uint64, C.c_char_p, C.c_size_t] + cu.cuMemcpyDtoH_v2.argtypes = [C.c_void_p, C.c_uint64, C.c_size_t] + cu.cuMemsetD8_v2.argtypes = [C.c_uint64, C.c_ubyte, C.c_size_t] + cu.cuMemFree_v2.argtypes = [C.c_uint64] + cu.cuLaunchKernel.argtypes = [C.c_void_p] + [C.c_uint] * 7 + [C.c_void_p, C.c_void_p, C.c_void_p] + init() + cmd, a = sys.argv[1], sys.argv[2:] + if cmd == 'ptx2cubin': + open(a[1], 'wb').write(ptx2cubin(open(a[0], 'rb').read())) + elif cmd == 'run': + src, kernel, grid, block, shared, in0, out_bytes, out = a[:8] + image = open(src, 'rb').read() + if src.endswith('.ptx'): + image = ptx2cubin(image) + ins = [open(p, 'rb').read() for p in [in0] + a[8:]] + open(out, 'wb').write(run(image, kernel, int(grid), int(block), int(shared), ins, int(out_bytes))) + else: + raise SystemExit(__doc__) diff --git a/dm/gfx/tools/ampere_orin/ops.py b/dm/gfx/tools/ampere_orin/ops.py new file mode 100644 index 0000000000..6ea50aace1 --- /dev/null +++ b/dm/gfx/tools/ampere_orin/ops.py @@ -0,0 +1,237 @@ +"""The ops the batch checks: each a PTX body over 32-bit inputs %i0.. that writes 32-bit outputs %o0... + +Input kinds (gen.py makes the values): f = f32, h = f16x2, b = bf16x2, u = 32-bit integer, s = shift or bit count, l = lane index, p = predicate-ish bit. +Output kinds (compare.py reads them): f, h, b, u. An op is (name, family, inputs, outputs, body).""" + +OPS = [] + + +def op(name, family, ins, outs, body): + OPS.append(dict(name=name, family=family, ins=ins, outs=outs, body=body)) + + +def fbin(o, extra=''): # f32 a op b + return f'mov.b32 %f0, %i0; mov.b32 %f1, %i1; {o} %f2, %f0, %f1; mov.b32 %o0, %f2;' + + +def fun(o): # f32 op a + return f'mov.b32 %f0, %i0; {o} %f2, %f0; mov.b32 %o0, %f2;' + + +def fter(o): + return f'mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; {o} %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;' + + +def hbin(o): # f16x2 a op b + return f'mov.b32 %x0, %i0; mov.b32 %x1, %i1; {o} %x2, %x0, %x1; mov.b32 %o0, %x2;' + + +def hter(o): + return f'mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; {o} %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;' + + +# --- f32 arithmetic (FADD / FMUL / FFMA / FMNMX), every rounding mode, ftz and sat +for rnd in ('rn', 'rz', 'rm', 'rp'): + op(f'fadd_{rnd}', 'f32 arith', 'ff', 'f', fbin(f'add.{rnd}.f32')) + op(f'fmul_{rnd}', 'f32 arith', 'ff', 'f', fbin(f'mul.{rnd}.f32')) + op(f'ffma_{rnd}', 'f32 arith', 'fff', 'f', fter(f'fma.{rnd}.f32')) +op('fadd_ftz', 'f32 arith', 'ff', 'f', fbin('add.rn.ftz.f32')) +op('fadd_sat', 'f32 arith', 'ff', 'f', fbin('add.rn.sat.f32')) +op('fsub', 'f32 arith', 'ff', 'f', fbin('sub.rn.f32')) +op('fmul_ftz', 'f32 arith', 'ff', 'f', fbin('mul.rn.ftz.f32')) +op('fmul_sat', 'f32 arith', 'ff', 'f', fbin('mul.rn.sat.f32')) +op('ffma_ftz', 'f32 arith', 'fff', 'f', fter('fma.rn.ftz.f32')) +op('ffma_sat', 'f32 arith', 'fff', 'f', fter('fma.rn.sat.f32')) +op('ffma_ftz_sat', 'f32 arith', 'fff', 'f', fter('fma.rn.ftz.sat.f32')) +op('fadd_abs_neg', 'f32 arith', 'ff', 'f', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; abs.f32 %f2, %f0; neg.f32 %f3, %f1; add.rn.f32 %f4, %f2, %f3; mov.b32 %o0, %f4;') +op('fmul_by_imm', 'f32 arith', 'f', 'f', 'mov.b32 %f0, %i0; mul.rn.f32 %f2, %f0, 0f3EAAAAAB; mov.b32 %o0, %f2;') +op('fmin', 'f32 arith', 'ff', 'f', fbin('min.f32')) +op('fmax', 'f32 arith', 'ff', 'f', fbin('max.f32')) +op('fmin_ftz', 'f32 arith', 'ff', 'f', fbin('min.ftz.f32')) +op('fmax_nan', 'f32 arith', 'ff', 'f', fbin('max.NaN.f32')) +op('fmin_nan', 'f32 arith', 'ff', 'f', fbin('min.NaN.f32')) +op('fcopysign', 'f32 arith', 'ff', 'f', fbin('copysign.f32')) +op('fabs', 'f32 arith', 'f', 'f', fun('abs.f32')) +op('fneg', 'f32 arith', 'f', 'f', fun('neg.f32')) + +# --- f32 compares (FSETP / FSET), many predicates in one kernel +CMPS = ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'equ', 'neu', 'ltu', 'leu', 'gtu', 'geu', 'num', 'nan') +op('fsetp', 'f32 compare', 'ff', 'u' * len(CMPS), 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; ' + ' '.join( + f'setp.{c}.f32 %p0, %f0, %f1; selp.u32 %o{k}, 1, 0, %p0;' for k, c in enumerate(CMPS))) +op('fsetp_ftz', 'f32 compare', 'ff', 'uuuu', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; ' + ' '.join( + f'setp.{c}.ftz.f32 %p0, %f0, %f1; selp.u32 %o{k}, 1, 0, %p0;' for k, c in enumerate(('eq', 'lt', 'gtu', 'ne')))) +op('fset_bf', 'f32 compare', 'ff', 'ff', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; set.lt.f32.f32 %f2, %f0, %f1; set.neu.f32.f32 %f3, %f0, %f1; mov.b32 %o0, %f2; mov.b32 %o1, %f3;') +op('fset_u32', 'f32 compare', 'ff', 'uu', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; set.ge.u32.f32 %o0, %f0, %f1; set.nan.u32.f32 %o1, %f0, %f1;') +op('ftestp', 'f32 compare', 'f', 'uuuuu', 'mov.b32 %f0, %i0; ' + ' '.join( + f'testp.{t}.f32 %p0, %f0; selp.u32 %o{k}, 1, 0, %p0;' for k, t in enumerate(('finite', 'infinite', 'number', 'normal', 'subnormal')))) +op('fsel_cmp', 'f32 compare', 'fff', 'f', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; setp.gt.f32 %p0, %f0, 0f00000000; selp.f32 %f2, %f1, %f3, %p0; mov.b32 %o0, %f2;') + +# --- MUFU and the precise sequences built on it +for o in ('ex2.approx.f32', 'ex2.approx.ftz.f32', 'lg2.approx.f32', 'lg2.approx.ftz.f32', 'rcp.approx.f32', 'rcp.approx.ftz.f32', 'rsqrt.approx.f32', + 'rsqrt.approx.ftz.f32', 'sqrt.approx.f32', 'sqrt.approx.ftz.f32', 'sin.approx.f32', 'cos.approx.f32', 'tanh.approx.f32'): + op('mufu_' + o.replace('.approx', '').replace('.f32', '').replace('.', '_'), 'mufu', 'f', 'f', fun(o)) +op('mufu_ex2_f16x2', 'mufu', 'h', 'h', 'mov.b32 %x0, %i0; ex2.approx.f16x2 %x2, %x0; mov.b32 %o0, %x2;') +op('mufu_tanh_f16x2', 'mufu', 'h', 'h', 'mov.b32 %x0, %i0; tanh.approx.f16x2 %x2, %x0; mov.b32 %o0, %x2;') +op('fdiv_rn', 'f32 precise', 'ff', 'f', fbin('div.rn.f32')) +op('fdiv_full', 'f32 precise', 'ff', 'f', fbin('div.full.f32')) +op('fdiv_approx', 'f32 precise', 'ff', 'f', fbin('div.approx.f32')) +op('frcp_rn', 'f32 precise', 'f', 'f', fun('rcp.rn.f32')) +op('fsqrt_rn', 'f32 precise', 'f', 'f', fun('sqrt.rn.f32')) + +# --- conversions (F2I, I2F, F2F, FRND, F2FP, I2I) +for rnd in ('rni', 'rzi', 'rmi', 'rpi'): + op(f'f2i_s32_{rnd}', 'convert', 'f', 'u', f'mov.b32 %f0, %i0; cvt.{rnd}.s32.f32 %o0, %f0;') + op(f'frnd_{rnd}', 'convert', 'f', 'f', f'mov.b32 %f0, %i0; cvt.{rnd}.f32.f32 %f2, %f0; mov.b32 %o0, %f2;') +op('f2i_u32_rzi', 'convert', 'f', 'u', 'mov.b32 %f0, %i0; cvt.rzi.u32.f32 %o0, %f0;') +op('f2i_s32_ftz', 'convert', 'f', 'u', 'mov.b32 %f0, %i0; cvt.rzi.ftz.s32.f32 %o0, %f0;') +op('f2i_s16_sat', 'convert', 'f', 'u', 'mov.b32 %f0, %i0; cvt.rzi.sat.s16.f32 %w0, %f0; cvt.s32.s16 %o0, %w0;') +op('f2i_u8_sat', 'convert', 'f', 'u', 'mov.b32 %f0, %i0; cvt.rni.sat.u8.f32 %w0, %f0; cvt.u32.u8 %o0, %w0;') +op('f2i_s64', 'convert', 'f', 'uu', 'mov.b32 %f0, %i0; cvt.rzi.s64.f32 %d0, %f0; mov.b64 {%o0, %o1}, %d0;') +for rnd in ('rn', 'rz', 'rm', 'rp'): + op(f'i2f_s32_{rnd}', 'convert', 'u', 'f', f'cvt.{rnd}.f32.s32 %f2, %i0; mov.b32 %o0, %f2;') +op('i2f_u32_rn', 'convert', 'u', 'f', 'cvt.rn.f32.u32 %f2, %i0; mov.b32 %o0, %f2;') +op('i2f_s16', 'convert', 'u', 'f', 'cvt.u16.u32 %w0, %i0; cvt.rn.f32.s16 %f2, %w0; mov.b32 %o0, %f2;') +op('i2f_s64', 'convert', 'uu', 'f', 'mov.b64 %d0, {%i0, %i1}; cvt.rn.f32.s64 %f2, %d0; mov.b32 %o0, %f2;') +op('f2f_f16_rn', 'convert', 'f', 'h', 'mov.b32 %f0, %i0; cvt.rn.f16.f32 %y0, %f0; mov.b16 %w0, %y0; cvt.u32.u16 %o0, %w0;') +op('f2f_f16_rz', 'convert', 'f', 'h', 'mov.b32 %f0, %i0; cvt.rz.f16.f32 %y0, %f0; mov.b16 %w0, %y0; cvt.u32.u16 %o0, %w0;') +op('f2f_f32_from_f16', 'convert', 'h', 'ff', 'mov.b32 {%y0, %y1}, %i0; cvt.f32.f16 %f2, %y0; cvt.f32.f16 %f3, %y1; mov.b32 %o0, %f2; mov.b32 %o1, %f3;') +op('f2fp_pack_f16x2', 'convert', 'ff', 'h', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.f16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;') +op('f2fp_pack_f16x2_relu', 'convert', 'ff', 'h', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.relu.f16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;') +op('f2fp_pack_bf16x2', 'convert', 'ff', 'b', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.bf16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;') +op('f2f_tf32', 'convert', 'f', 'u', 'mov.b32 %f0, %i0; cvt.rna.tf32.f32 %o0, %f0;') +op('i2i_s8_sat', 'convert', 'u', 'u', 'cvt.sat.s8.s32 %w0, %i0; cvt.s32.s8 %o0, %w0;') +op('i2i_u16_sat', 'convert', 'u', 'u', 'cvt.sat.u16.s32 %w0, %i0; cvt.u32.u16 %o0, %w0;') +op('sext_s8', 'convert', 'u', 'u', 'cvt.s32.s8 %o0, %i0;') + +# --- f16x2 / bf16x2 (HADD2 / HMUL2 / HFMA2 / HMNMX2 / HSETP2 / HSET2) +op('hadd2', 'f16x2', 'hh', 'h', hbin('add.rn.f16x2')) +op('hadd2_ftz', 'f16x2', 'hh', 'h', hbin('add.rn.ftz.f16x2')) +op('hadd2_sat', 'f16x2', 'hh', 'h', hbin('add.rn.sat.f16x2')) +op('hsub2', 'f16x2', 'hh', 'h', hbin('sub.rn.f16x2')) +op('hmul2', 'f16x2', 'hh', 'h', hbin('mul.rn.f16x2')) +op('hmul2_ftz', 'f16x2', 'hh', 'h', hbin('mul.rn.ftz.f16x2')) +op('hfma2', 'f16x2', 'hhh', 'h', hter('fma.rn.f16x2')) +op('hfma2_ftz', 'f16x2', 'hhh', 'h', hter('fma.rn.ftz.f16x2')) +op('hfma2_sat', 'f16x2', 'hhh', 'h', hter('fma.rn.sat.f16x2')) +op('hfma2_relu', 'f16x2', 'hhh', 'h', hter('fma.rn.relu.f16x2')) +op('hmin2', 'f16x2', 'hh', 'h', hbin('min.f16x2')) +op('hmax2', 'f16x2', 'hh', 'h', hbin('max.f16x2')) +op('hmin2_nan', 'f16x2', 'hh', 'h', hbin('min.NaN.f16x2')) +op('hneg2', 'f16x2', 'h', 'h', 'mov.b32 %x0, %i0; neg.f16x2 %x2, %x0; mov.b32 %o0, %x2;') +op('habs2', 'f16x2', 'h', 'h', 'mov.b32 %x0, %i0; abs.f16x2 %x2, %x0; mov.b32 %o0, %x2;') +op('hadd2_imm', 'f16x2', 'h', 'h', 'mov.b32 %x0, %i0; mov.b32 %x1, 0x3C00BC00; add.rn.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;') +op('hadd_f16', 'f16x2', 'hh', 'hh', 'mov.b32 {%y0, %y1}, %i0; mov.b32 {%y2, %y3}, %i1; add.rn.f16 %y4, %y0, %y2; mul.rn.f16 %y5, %y1, %y3; mov.b16 %w0, %y4; cvt.u32.u16 %o0, %w0; mov.b16 %w1, %y5; cvt.u32.u16 %o1, %w1;') +op('hsetp2', 'f16x2', 'hh', 'uuuuuuuu', 'mov.b32 %x0, %i0; mov.b32 %x1, %i1; ' + ' '.join( + f'setp.{c}.f16x2 %p0|%p1, %x0, %x1; selp.u32 %o{2*k}, 1, 0, %p0; selp.u32 %o{2*k+1}, 1, 0, %p1;' for k, c in enumerate(('lt', 'eq', 'gtu', 'nan')))) +op('hset2', 'f16x2', 'hh', 'hu', 'mov.b32 %x0, %i0; mov.b32 %x1, %i1; set.lt.f16x2.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2; set.geu.u32.f16x2 %o1, %x0, %x1;') +op('bfma2', 'bf16x2', 'bbb', 'b', hter('fma.rn.bf16x2')) +op('bmin2', 'bf16x2', 'bb', 'b', hbin('min.bf16x2')) +op('bmax2_nan', 'bf16x2', 'bb', 'b', hbin('max.NaN.bf16x2')) + +# --- integer ALU +op('iadd', 'int', 'uu', 'u', 'add.u32 %o0, %i0, %i1;') +op('iadd3', 'int', 'uuu', 'u', 'add.u32 %t0, %i0, %i1; sub.u32 %o0, %t0, %i2;') +op('iadd_sat', 'int', 'uu', 'u', 'add.sat.s32 %o0, %i0, %i1;') +op('isub_sat', 'int', 'uu', 'u', 'sub.sat.s32 %o0, %i0, %i1;') +op('iadd64', 'int', 'uuuu', 'uu', 'mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; add.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;') +op('isub64', 'int', 'uuuu', 'uu', 'mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; sub.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;') +op('imul_lo', 'int', 'uu', 'u', 'mul.lo.u32 %o0, %i0, %i1;') +op('imul_hi_u32', 'int', 'uu', 'u', 'mul.hi.u32 %o0, %i0, %i1;') +op('imul_hi_s32', 'int', 'uu', 'u', 'mul.hi.s32 %o0, %i0, %i1;') +op('imul_wide_s32', 'int', 'uu', 'uu', 'mul.wide.s32 %d0, %i0, %i1; mov.b64 {%o0, %o1}, %d0;') +op('imul_wide_u32', 'int', 'uu', 'uu', 'mul.wide.u32 %d0, %i0, %i1; mov.b64 {%o0, %o1}, %d0;') +op('imad_lo', 'int', 'uuu', 'u', 'mad.lo.u32 %o0, %i0, %i1, %i2;') +op('imad_hi_s32', 'int', 'uuu', 'u', 'mad.hi.s32 %o0, %i0, %i1, %i2;') +op('imul64', 'int', 'uuuu', 'uu', 'mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; mul.lo.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;') +op('imul24', 'int', 'uu', 'u', 'mul24.lo.s32 %o0, %i0, %i1;') +op('imin_s32', 'int', 'uu', 'u', 'min.s32 %o0, %i0, %i1;') +op('imax_u32', 'int', 'uu', 'u', 'max.u32 %o0, %i0, %i1;') +op('imin_s64', 'int', 'uuuu', 'uu', 'mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; min.s64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;') +op('iabs', 'int', 'u', 'u', 'abs.s32 %o0, %i0;') +op('ineg', 'int', 'u', 'u', 'neg.s32 %o0, %i0;') +op('isad', 'int', 'uuu', 'u', 'sad.u32 %o0, %i0, %i1, %i2;') +op('idiv_u32', 'int div', 'uu', 'u', 'div.u32 %o0, %i0, %i1;') +op('irem_u32', 'int div', 'uu', 'u', 'rem.u32 %o0, %i0, %i1;') +op('idiv_s32', 'int div', 'uu', 'u', 'div.s32 %o0, %i0, %i1;') +op('irem_s32', 'int div', 'uu', 'u', 'rem.s32 %o0, %i0, %i1;') +op('idiv_u32_by_7', 'int div', 'u', 'u', 'div.u32 %o0, %i0, 7;') +op('isetp', 'int compare', 'uu', 'uuuuuu', ' '.join( + f'setp.{c} %p0, %i0, %i1; selp.u32 %o{k}, 1, 0, %p0;' for k, c in enumerate(('lt.s32', 'lt.u32', 'ge.s32', 'hi.u32', 'eq.s32', 'ne.u32')))) +op('isetp_and', 'int compare', 'uuuu', 'uu', 'setp.lt.s32 %p1, %i2, %i3; setp.lt.and.s32 %p0, %i0, %i1, %p1; selp.u32 %o0, 1, 0, %p0; ' + 'setp.ge.or.u32 %p2, %i0, %i1, %p1; selp.u32 %o1, 1, 0, %p2;') +op('isetp64', 'int compare', 'uuuu', 'uu', 'mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; setp.lt.s64 %p0, %d0, %d1; selp.u32 %o0, 1, 0, %p0; ' + 'setp.lt.u64 %p1, %d0, %d1; selp.u32 %o1, 1, 0, %p1;') +op('plop3', 'int compare', 'uuu', 'u', 'setp.ne.u32 %p0, %i0, 0; setp.ne.u32 %p1, %i1, 0; setp.ne.u32 %p2, %i2, 0; xor.pred %p3, %p0, %p1; or.pred %p3, %p3, %p2; ' + 'not.pred %p3, %p3; selp.u32 %o0, 1, 0, %p3;') + +# --- bits +op('shl', 'bits', 'us', 'u', 'shl.b32 %o0, %i0, %i1;') +op('shr_u32', 'bits', 'us', 'u', 'shr.u32 %o0, %i0, %i1;') +op('shr_s32', 'bits', 'us', 'u', 'shr.s32 %o0, %i0, %i1;') +op('shl_imm', 'bits', 'u', 'uu', 'shl.b32 %o0, %i0, 7; shr.s32 %o1, %i0, 13;') +op('shf_l_wrap', 'bits', 'uus', 'u', 'shf.l.wrap.b32 %o0, %i0, %i1, %i2;') +op('shf_l_clamp', 'bits', 'uus', 'u', 'shf.l.clamp.b32 %o0, %i0, %i1, %i2;') +op('shf_r_wrap', 'bits', 'uus', 'u', 'shf.r.wrap.b32 %o0, %i0, %i1, %i2;') +op('shf_r_clamp', 'bits', 'uus', 'u', 'shf.r.clamp.b32 %o0, %i0, %i1, %i2;') +op('shl64', 'bits', 'uus', 'uu', 'mov.b64 %d0, {%i0, %i1}; shl.b64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;') +op('shr64_u', 'bits', 'uus', 'uu', 'mov.b64 %d0, {%i0, %i1}; shr.u64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;') +op('shr64_s', 'bits', 'uus', 'uu', 'mov.b64 %d0, {%i0, %i1}; shr.s64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;') +LUTS = (0x96, 0xe8, 0xca, 0x01, 0x80, 0x6a, 0xf8, 0x1e, 0xb4, 0x3c) +op('lop3', 'bits', 'uuu', 'u' * len(LUTS), ' '.join(f'lop3.b32 %o{k}, %i0, %i1, %i2, {hex(l)};' for k, l in enumerate(LUTS))) +op('logic', 'bits', 'uu', 'uuuu', 'and.b32 %o0, %i0, %i1; or.b32 %o1, %i0, %i1; xor.b32 %o2, %i0, %i1; not.b32 %t0, %i0; and.b32 %o3, %t0, %i1;') +PRMT = ('', '.f4e', '.b4e', '.rc8', '.ecl', '.ecr', '.rc16') +op('prmt', 'bits', 'uuu', 'u' * len(PRMT), ' '.join(f'prmt.b32{m} %o{k}, %i0, %i1, %i2;' for k, m in enumerate(PRMT))) +op('prmt_imm', 'bits', 'uu', 'uuu', 'prmt.b32 %o0, %i0, %i1, 0x3210; prmt.b32 %o1, %i0, %i1, 0x5140; prmt.b32 %o2, %i0, %i1, 0x8880;') +op('bfe_u32', 'bits', 'uss', 'u', 'bfe.u32 %o0, %i0, %i1, %i2;') +op('bfe_s32', 'bits', 'uss', 'u', 'bfe.s32 %o0, %i0, %i1, %i2;') +op('bfi', 'bits', 'uuss', 'u', 'bfi.b32 %o0, %i0, %i1, %i2, %i3;') +op('bmsk_clamp', 'bits', 'ss', 'u', 'bmsk.clamp.b32 %o0, %i0, %i1;') +op('bmsk_wrap', 'bits', 'ss', 'u', 'bmsk.wrap.b32 %o0, %i0, %i1;') +op('szext_s', 'bits', 'us', 'u', 'szext.clamp.s32 %o0, %i0, %i1;') +op('popc', 'bits', 'u', 'u', 'popc.b32 %o0, %i0;') +op('popc64', 'bits', 'uu', 'u', 'mov.b64 %d0, {%i0, %i1}; popc.b64 %o0, %d0;') +op('clz', 'bits', 'u', 'u', 'clz.b32 %o0, %i0;') +op('clz64', 'bits', 'uu', 'u', 'mov.b64 %d0, {%i0, %i1}; clz.b64 %o0, %d0;') +op('bfind_u32', 'bits', 'u', 'u', 'bfind.u32 %o0, %i0;') +op('bfind_s32', 'bits', 'u', 'u', 'bfind.s32 %o0, %i0;') +op('bfind_shiftamt', 'bits', 'u', 'u', 'bfind.shiftamt.u32 %o0, %i0;') +op('brev', 'bits', 'u', 'u', 'brev.b32 %o0, %i0;') +op('fns', 'bits', 'uus', 'u', 'and.b32 %t0, %i2, 31; fns.b32 %o0, %i0, %i1, %t0;') +op('dp4a_u32', 'bits', 'uuu', 'u', 'dp4a.u32.u32 %o0, %i0, %i1, %i2;') +op('dp4a_s32', 'bits', 'uuu', 'u', 'dp4a.s32.s32 %o0, %i0, %i1, %i2;') +op('dp4a_su', 'bits', 'uuu', 'u', 'dp4a.s32.u32 %o0, %i0, %i1, %i2;') +op('dp2a_lo', 'bits', 'uuu', 'u', 'dp2a.lo.s32.s32 %o0, %i0, %i1, %i2;') +op('dp2a_hi', 'bits', 'uuu', 'u', 'dp2a.hi.u32.u32 %o0, %i0, %i1, %i2;') + +# --- warp (SHFL / VOTE / MATCH / REDUX): 32 threads to a group on both sides, so a lane's neighbours are the same +op('shfl_idx', 'warp', 'ul', 'u', 'shfl.sync.idx.b32 %o0, %i0, %i1, 0x1f, 0xffffffff;') +op('shfl_up', 'warp', 'us', 'uu', 'and.b32 %t0, %i1, 31; shfl.sync.up.b32 %o0|%p0, %i0, %t0, 0, 0xffffffff; selp.u32 %o1, 1, 0, %p0;') +op('shfl_down', 'warp', 'us', 'uu', 'and.b32 %t0, %i1, 31; shfl.sync.down.b32 %o0|%p0, %i0, %t0, 0x1f, 0xffffffff; selp.u32 %o1, 1, 0, %p0;') +op('shfl_bfly', 'warp', 'ul', 'u', 'shfl.sync.bfly.b32 %o0, %i0, %i1, 0x1f, 0xffffffff;') +op('shfl_bfly_imm', 'warp', 'u', 'u', 'shfl.sync.bfly.b32 %o0, %i0, 1, 0x1f, 0xffffffff;') +op('shfl_idx_seg', 'warp', 'ul', 'u', 'shfl.sync.idx.b32 %o0, %i0, %i1, 0x1807, 0xffffffff;') +op('vote', 'warp', 'p', 'uuuu', 'and.b32 %t0, %i0, 1; setp.ne.u32 %p0, %t0, 0; vote.sync.ballot.b32 %o0, %p0, 0xffffffff; vote.sync.any.pred %p1, %p0, 0xffffffff; ' + 'selp.u32 %o1, 1, 0, %p1; vote.sync.all.pred %p2, %p0, 0xffffffff; selp.u32 %o2, 1, 0, %p2; vote.sync.uni.pred %p3, %p0, 0xffffffff; selp.u32 %o3, 1, 0, %p3;') +op('activemask', 'warp', 'p', 'u', 'activemask.b32 %o0;') +op('match_any', 'warp', 'p', 'u', 'and.b32 %t0, %i0, 3; match.any.sync.b32 %o0, %t0, 0xffffffff;') +op('redux_add', 'warp', 'u', 'u', 'redux.sync.add.u32 %o0, %i0, 0xffffffff;') +op('redux_min_s32', 'warp', 'u', 'u', 'redux.sync.min.s32 %o0, %i0, 0xffffffff;') +op('redux_or', 'warp', 'u', 'u', 'redux.sync.or.b32 %o0, %i0, 0xffffffff;') + +# --- tensor (HMMA / IMMA): each thread holds its fragments, as the PTX layouts give them +op('hmma_16816_f16', 'tensor', 'HHHHHHHH', 'hh', 'mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%o0, %o1}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%i6, %i7};') +op('hmma_16816_f32', 'tensor', 'HHHHHHFFFF', 'ffff', 'mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; ' + 'mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; ' + 'mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;') +op('hmma_1688_f16', 'tensor', 'HHHHH', 'hh', 'mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%o0, %o1}, {%i0, %i1}, {%i2}, {%i3, %i4};') +op('hmma_16816_bf16', 'tensor', 'BBBBBBFFFF', 'ffff', 'mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; ' + 'mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; ' + 'mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;') +op('hmma_1688_tf32', 'tensor', 'FFFFFFFFFF', 'ffff', 'mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; ' + 'mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; ' + 'mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;') +op('imma_16832_s8', 'tensor', 'UUUUUUUUUU', 'uuuu', 'mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%o0, %o1, %o2, %o3}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%i6, %i7, %i8, %i9};') + +# --- f64 (DADD / DMUL / DFMA): rare in shaders, here for the coverage list +op('dadd', 'f64', 'uuuu', 'uu', 'mov.b64 %g0, {%i0, %i1}; mov.b64 %g1, {%i2, %i3}; add.rn.f64 %g2, %g0, %g1; mov.b64 {%o0, %o1}, %g2;') +op('dmul', 'f64', 'uuuu', 'uu', 'mov.b64 %g0, {%i0, %i1}; mov.b64 %g1, {%i2, %i3}; mul.rn.f64 %g2, %g0, %g1; mov.b64 {%o0, %o1}, %g2;') diff --git a/dm/gfx/tools/ampere_orin/prog.py b/dm/gfx/tools/ampere_orin/prog.py new file mode 100644 index 0000000000..233325e55b --- /dev/null +++ b/dm/gfx/tools/ampere_orin/prog.py @@ -0,0 +1,163 @@ +"""The Metal side's program, made from the very SASS the Orin ran (stdlib only, so the Jetson can run it and disassemble the result). + +The op's kernel is k(in, out): it loads its NIN input words from in[gid * NIN + k] (LDG), computes, and stores NOUT words to out[gid * NOUT + k] (STG). +On the Metal side the translator gets the same instructions, where: + - two instructions go first: S2R Rb = SR_TID.X; IMAD Rb = Rb * W * 4 (W = NIN + NOUT words a thread, Rb a register the kernel does not use) + - each LDG Rd, [Ra + 4k] becomes LDS Rd, [Rb + 4k]; each STG [Ra + 4k], Rs becomes STS [Rb + 4 (NIN + k)], Rs (same guard and control bits) + - each instruction that reads the CUDA parameter bank c[0x0] (the stack pointer, the global window, the two pointers) becomes a NOP +so a thread's words lie at shared memory word tid * W: its inputs, then its outputs. Branches are relative, so the two inserted instructions move nothing.""" +import re +import struct + +CTRL = ~((1 << 41) - 1) & (2 ** 64 - 1) # the control bits (stall, yield, barriers, wait mask, reuse): bits 105..127 of the instruction, 41..63 of its high word +WAIT_ALL = 0x3f << 52 + + +def text_section(cubin, kernel='k'): + shoff, = struct.unpack_from(' instruction text, from nvdisasm -hex""" + out = {} + for m in re.finditer(r'/\*([0-9a-f]{4,})\*/\s+(.*?)\s*;\s*/\* 0x', sass): + out[int(m.group(1), 16)] = m.group(2) + return out + + +def rewrite(cubin, sass, nin, nout): + code = text_section(cubin) + text = sass_lines(sass) + insts = [struct.unpack_from('= 255: + raise ValueError('no free register') + w = nin + nout + out, notes, seen_in, seen_out = [], [], set(), set() + # The driver's JIT starts every kernel with a one-time convergence barrier set-up: ISETP P0 on UR2; @P0 BRA past it; BMOV B0..B15. Offline-compiled + # shaders (the games') have none, so it goes, up to the branch's target + skip = 0 + if text.get(0, '').startswith('ISETP.NE.U32.AND P0, PT, RZ, UR2') and text.get(16, '').startswith('@P0 BRA'): + rel = (insts[1][0] >> 32) & 0xffffffff # BRA's offset from the next instruction, in bytes + skip = (32 + rel) // 16 + notes.append(f'nop 0x0000..{skip * 16:#06x}: the JIT\'s barrier set-up') + dead = address_only(insts, text, skip) + for n in sorted(dead): + notes.append(f'nop {n * 16:#06x} (feeds only the removed addressing): {text.get(n * 16, "")}') + out.append((0x0000000000007919 | rb << 16, 0x000e280000002100)) # S2R Rb, SR_TID.X (write barrier 0) + out.append((0x0000000000007824 | rb << 16 | rb << 24 | (w * 4) << 32, 0x000fe200078e00ff | WAIT_ALL)) # IMAD Rb, Rb, W*4, RZ + for n, (lo, hi) in enumerate(insts): + t = text.get(n * 16, '') + opc = lo & 0xfff + guard = lo & 0xf000 + off = (lo >> 40) & 0xffffff + if n < skip: + out.append((0x7918, hi & CTRL)) + elif opc == 0x981: # LDG + if not re.match(r'(@!?U?P\w+ )?LDG\.E(\.STRONG\.\w+)? R\d+, \[R\d+\.64(\+0x[0-9a-f]+)?\]', t): + raise ValueError(f'unexpected load: {t}') + k = off // 4 + if off % 4 or not 0 <= k < nin: + raise ValueError(f'load offset {off:#x}: {t}') + seen_in.add(k) + rd = (lo >> 16) & 0xff + out.append((0x984 | guard | rd << 16 | rb << 24 | (k * 4) << 40, 0x0800 | (hi & CTRL))) + elif opc == 0x986: # STG + if not re.match(r'(@!?U?P\w+ )?STG\.E(\.STRONG\.\w+)? \[R\d+\.64(\+0x[0-9a-f]+)?\], R\d+', t): + raise ValueError(f'unexpected store: {t}') + k = off // 4 + if off % 4 or not 0 <= k < nout: + raise ValueError(f'store offset {off:#x}: {t}') + seen_out.add(k) + rs = (lo >> 32) & 0xff + out.append((0x388 | guard | rb << 24 | rs << 32 | ((nin + k) * 4) << 40, 0x0800 | (hi & CTRL))) + elif n in dead: + out.append((0x7918, hi & CTRL)) + else: + if re.search(r'c\[0x[1-9a-f]\]', t): + notes.append(f'constant bank read kept {n * 16:#06x}: {t}') + out.append((lo, hi)) + if seen_in != set(range(nin)) or seen_out != set(range(nout)): + raise ValueError(f'loads {sorted(seen_in)} of {nin}, stores {sorted(seen_out)} of {nout}') + return b''.join(struct.pack(' 3) { throw AFail(context, "I2F from 64 bits other than a register pair to an F32"); } Operand lo = x.Rb >= RegisterConsts.RegisterZeroIndex ? Const(0) : Register(x.Rb, RegisterType.Gpr); Operand hi = x.Rb + 1 >= RegisterConsts.RegisterZeroIndex ? Const(0) : Register(x.Rb + 1, RegisterType.Gpr); - Operand fh = x.Bit(74) ? context.IConvertS32ToFP32(hi) : context.IConvertU32ToFP32(hi); + Operand neg = null; + + if (x.Bit(74)) + { + // The magnitude of a negative value: the 64-bit two's complement, with the carry out of the low word when it is zero. -2^63 stays 2^63 read unsigned + neg = context.ICompareLess(hi, Const(0)); + Operand nhi = context.ConditionalSelect(context.ICompareEqual(lo, Const(0)), context.INegate(hi), context.BitwiseNot(hi)); + hi = context.ConditionalSelect(neg, nhi, hi); + lo = context.ConditionalSelect(neg, context.INegate(lo), lo); + } + + // top: the word that holds the leading one, bottom: the word below it; s: the leading zeros of top (0..31). The 64 bits shifted left by s give h (the top + // word, its bit 31 set) and l (what is left of the low word). (bottom >> 1) >> (31 - s) is bottom >> (32 - s) with no shift by 32 + Operand hz = context.ICompareEqual(hi, Const(0)); + Operand top = context.ConditionalSelect(hz, lo, hi); + Operand bottom = context.ConditionalSelect(hz, Const(0), lo); + Operand msb = context.FindMSBU32(top); + Operand s = context.ISubtract(Const(31), msb); + Operand h = context.BitwiseOr(context.ShiftLeft(top, s), context.ShiftRightU32(context.ShiftRightU32(bottom, Const(1)), msb)); + Operand l = context.ShiftLeft(bottom, s); + + // The leading one sits at bit 32 + msb (or msb), so the biased exponent is that + 127. Its bit 23 of h >> 8 adds one to the exponent field, hence 126. + // A rounding carry out of the mantissa moves into the exponent, as it should + Operand exp = context.IAdd(msb, context.ConditionalSelect(hz, Const(126), Const(126 + 32))); + Operand bits = context.IAdd(context.ShiftLeft(exp, Const(23)), context.ShiftRightU32(h, Const(8))); + Operand lNz = context.ICompareNotEqual(l, Const(0)); + Operand inc; + + if (rnd == 0) + { + // Nearest even: the round bit (h bit 7) set, and a sticky bit below it or an odd mantissa (h bit 8) + Operand roundBit = context.ICompareNotEqual(context.BitwiseAnd(h, Const(0x80)), Const(0)); + Operand rest = context.BitwiseOr(context.ICompareNotEqual(context.BitwiseAnd(h, Const(0x17f)), Const(0)), lNz); + inc = context.BitwiseAnd(roundBit, rest); + } + else if (rnd == 3) + { + inc = null; // RZ: the magnitude truncated + } + else + { + // RM (1) rounds the magnitude up for a negative value, RP (2) for a positive one, when any bit was dropped + Operand inexact = context.BitwiseOr(context.ICompareNotEqual(context.BitwiseAnd(h, Const(0xff)), Const(0)), lNz); + inc = rnd == 1 + ? (neg == null ? null : context.BitwiseAnd(inexact, neg)) + : (neg == null ? inexact : context.BitwiseAnd(inexact, context.BitwiseNot(neg))); + } + + if (inc != null) + { + bits = context.ConditionalSelect(inc, context.IAdd(bits, Const(1)), bits); + } + + if (neg != null) + { + bits = context.BitwiseOr(bits, context.ConditionalSelect(neg, Const(unchecked((int)0x80000000)), Const(0))); + } - ASet(context, x, context.FPAdd(context.FPMultiply(fh, ConstF(4294967296.0f)), context.IConvertU32ToFP32(lo))); + // Zero has no leading one (FindMSBU32 gives -1) + ASet(context, x, context.ConditionalSelect(context.ICompareEqual(top, Const(0)), Const(0), bits)); } // I2FP.F32.S32 / .U32: signed at 74 diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index bc709640d5..636d124c7d 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -977,6 +977,8 @@ One line per symptom in the [subsystem references](../reference/): what you see - a Switch 2 game's movie or upscaler network output has dotted streaks, every other pixel of a few rows wrong, in other places each run, while its input is clean. Through agl's NDSR upscaler a few such pixels a frame grow into white blotches over the whole picture: Ocarina of Time's Kokiri Forest on an M4 Pro on chain 844bf1a5ea8f, with the game's own upscaler (the default); with `DM_UPSCALER=metalfx` in its place, no blotches. → [LDSM reads rows other lanes stored](../reference/shaders.md#ldsm-reads-rows-other-lanes-stored) - dotted horizontal rows across the picture at a regular spacing, in fog and light, with the game's upscaler and with `DM_UPSCALER=metalfx` alike (MetalFX replaces only `agl_ndsr.pck`'s shaders): Ocarina of Time in Kokiri Forest, Hyrule Field and the castle town since chain 23a1ea1db092. The rows are the tiles of agl's `agl_ndsr_vr.pck` / `agl_ndsr_vr_light.pck` networks (about 28.8 pixels of the 1536-wide input). → [An HMMA outside a chain overwrote the chain's accumulator](../reference/shaders.md#an-hmma-outside-a-chain-overwrote-the-chains-accumulator) - a Switch 2 game's movie or upscaler network gives values a few f16 steps off the console's, in about half the outputs of each HMMA. Nothing looks broken; it is the network's arithmetic, compounded layer by layer. → [HMMA rounds its f16 accumulator once per 8 products](../reference/shaders.md#hmma-rounds-its-f16-accumulator-once-per-8-products) +- none that shows on its own: a translated instruction gives other bits than the console's GPU for some inputs (a NaN operand, a lane index past 31, a directed rounding), and the picture is off only where a shader depends on them. The translator tests check the MSL a few encodings give, not the values the shader computes. → [Ampere instructions checked against an Ampere GPU](../reference/shaders.md#ampere-instructions-checked-against-an-ampere-gpu) +- a 64-bit integer converted to a float (`I2F.S64` / `.U64`, PTX `cvt.rn.f32.s64`) comes out one ulp off for some values, and far more for a negative value whose high word is -1 or -2, as when a small negative number is held in 64 bits: `0xffffffff_fa543872` gave 0xccb57900 where the GPU gives 0xccb578f2. → [I2F from a 64-bit integer rounds once](../reference/shaders.md#i2f-from-a-64-bit-integer-rounds-once) - a Switch 2 game's movies and the title behind them run below 60 fps on a Mac with a smaller GPU, and the GPU profile (`DM_NVN_GPUPROF`) shows almost all of the frame in a dozen compute dispatches of `nn::ndve`'s movie network: Ocarina of Time's title at about 24 fps and 38 ms of GPU a frame on an M1 Pro with 16 GPU cores, 24.8 ms on an M1 Max (issue 1375), 18.9 ms of compute on an M4 with 8 (HMMA chains keep their matrices in the SIMD-group); 11-14 ms of the 16.7 on an M6, an M4 Pro or an M5 Max. → [The NDVE movie network by the host](../reference/shaders.md#the-ndve-movie-network-by-the-host) - a vertex or compute shader reads zeros from NVN's driver segment `c0`. → [Driver-segment reads outside fragment shaders](../reference/shaders.md#driver-segment-reads-outside-fragment-shaders) - flickering hard-edged white tiles (tiled lighting with every tile row getting row 0's or other rows' lights). → [Compute `c0`: dispatch size and uniform buffer table](../reference/shaders.md#compute-c0-dispatch-size-and-uniform-buffer-table) diff --git a/docs/reference/shaders.md b/docs/reference/shaders.md index f562418563..36c1d4d02c 100644 --- a/docs/reference/shaders.md +++ b/docs/reference/shaders.md @@ -1474,6 +1474,24 @@ Finding, keying and translating shaders (`dm/gfx/tools/shader_split.py`, `dmsh_b - **Check:** `test_dmsh_hmma.py`'s GPU runs compare every output bit for bit with the two-block reference: main's body differs in 21,461 of them on an M5 Max and on an M6, the new one in none; `test_ndve_engine.py` keeps the host's network bit for bit with the helper. Isolated on an M5 Max, NDSR's `fa3033d05c0a41d6` takes 0.316 ms against 0.276 and `d99fbeeb53d38847` 0.446 against 0.393; their outputs change in 1.5 % and 0.8 % of halves. On an M6, a chain of 258 calls takes 2.56 ms against 1.69 for 2,048 threadgroups of 4 SIMD-groups, and the host network's seven layer kernels at the movie's sizes 2.94 ms against 2.85 (best of 100 each, summed). Not measured in a game. - **Found on:** [The Legend of Zelda: Ocarina of Time](../../dm/games/0400486024DEC000/README.md): its NDVE movie network and agl's NDSR upscaler (PR #1897). +### Ampere instructions checked against an Ampere GPU + +- **Symptom:** none that shows on its own: a translated instruction gives other bits than the console's GPU for some inputs (a NaN operand, a lane index past 31, a directed rounding), and the picture is off only where a shader depends on them. The translator tests check the MSL a few encodings give, not the values the shader computes. +- **Cause:** nothing compared the translated instructions' values with an Ampere GPU's, so a meaning read wrong from the encoding, or lost on the way through spirv-cross and Metal, went unseen until a game showed it. +- **Fix:** `test_dmsh_ampere_orin.py` runs about 190 operations, each the GPU compiler's own machine code for one PTX operation, through the whole chain: the translator with the build's options, spirv-cross with the chain's arguments, every MSL rewrite `dmsh_build.py`'s `finish()` makes (read from its source, so a new rewrite is included), and the chain's Metal flags. On this Mac's GPU each takes the 1,024 input rows of its fixture, and every output word is compared bit for bit with what a Jetson Orin (sm_87, CUDA 12.6 driver) computed from the same rows. The inputs mix special values (signed zeros, denormals, infinities, NaNs with several payloads, the limits of each integer type, ties for the roundings), random bits and ordinary values; warp operations run 32 threads to a group on both sides, so a lane's neighbours are the same. `dm/gfx/tools/ampere_orin/capture.py` records the fixture on any Linux machine with an Ampere GPU and NVIDIA's driver: the driver compiles each op's PTX itself (no CUDA toolkit, no compiler on either side), runs it, and the same machine code is rewritten for the translator, its global loads and stores becoming shared memory ones and the CUDA parameter reads and the addressing they feed becoming NOPs (`prog.py`), which the device's `nvdisasm` checks. `ampere_orin/ampere_orin.sass` lists each program as the translator gets it. + - Allowed on every op: what Metal's f32 arithmetic itself gives, NaN bits (Ampere writes the canonical NaN, 0x7fffffff) and flushed f32 denormals (`compare.PLATFORM`). + - `KNOWN` lists the ops that still differ, each with its reason ([the tracking issue](https://github.com/seedds/nxnative/issues/2040)); `APPROX` the ones that run Metal's own approximation (MUFU, and the divisions built on it), which are reported and never fail. An op outside both lists that differs fails the test, and so does a `KNOWN` op that matches, so a fix takes its entry out. +- **Check:** `DMSH_TEST_XLATE=dm/gfx/tools/dmsh_xlate_bin/dmsh_xlate python3 dm/gfx/tests/test_dmsh_ampere_orin.py` (about 5 s on an M5 Max); it prints every op's differing lanes by cause. `AMPERE_ORIN_ONLY=` runs some, `AMPERE_ORIN_REPORT=` writes the results as JSON. To record again: `python3 dm/gfx/tools/ampere_orin/capture.py --host [--arch sm_86]`, with NVIDIA's `nvdisasm` (the `cuda_nvdisasm` redistributable) in `~/ampere_orin` on that machine. +- **Found on:** no game; a Jetson Orin's results. The HMMA comparison it extends is PR #1897's. + +### I2F from a 64-bit integer rounds once + +- **Symptom:** a 64-bit integer converted to a float (`I2F.S64` / `.U64`, PTX `cvt.rn.f32.s64`) comes out one ulp off for some values, and far more for a negative value whose high word is -1 or -2, as when a small negative number is held in 64 bits: `0xffffffff_fa543872` gave 0xccb57900 where the GPU gives 0xccb578f2. +- **Cause:** the translator has 32-bit integers only, so `AI2f64` converted the two words to float apart and added them, high word times 2^32 plus the low word: that rounds twice where the GPU rounds once, and for a negative value two large terms that nearly cancel leave the low word's rounding error in place. The rounding field (78..80) was ignored. +- **Fix:** `AI2f64` in `InstEmitAmpereAlu.cs` builds the float's bits in integer arithmetic, which Metal's fast math doesn't change: the magnitude of the 64-bit pair (with the carry between the words), its leading one found with `findMSB`, the 64 bits shifted so 24 of them make the mantissa, the next bit rounding and every bit below it sticky. It rounds to nearest even, or `.RM` / `.RP` / `.RZ` when the instruction asks, and a carry out of the mantissa moves into the exponent. About 35 more lines of MSL for each I2F from 64 bits, all integer operations; a model of the same steps matched an exact reference on 200,000 random pairs in all four roundings, signed and unsigned. +- **Check:** `test_dmsh_ampere_orin.py`, op `i2f_s64`: 16 of 1,024 lanes differed (up to 2^30 ulp), none with the fix, and it leaves `KNOWN`. The fixture has no directed rounding or unsigned source for this instruction, so those rest on the model. +- **Found on:** no game; a Jetson Orin's results ([Ampere instructions checked against an Ampere GPU](#ampere-instructions-checked-against-an-ampere-gpu)). + ### The NDVE movie network by the host - **Symptom:** a Switch 2 game's movies and the title behind them run below 60 fps on a Mac with a smaller GPU, and the GPU profile (`DM_NVN_GPUPROF`) shows almost all of the frame in a dozen compute dispatches of `nn::ndve`'s movie network: Ocarina of Time's title at about 24 fps and 38 ms of GPU a frame on an M1 Pro with 16 GPU cores, 24.8 ms on an M1 Max ([issue 1375](https://github.com/seedds/nxnative/issues/1375)), 18.9 ms of compute on an M4 with 8 ([HMMA chains keep their matrices in the SIMD-group](#hmma-chains-keep-their-matrices-in-the-simd-group)); 11-14 ms of the 16.7 on an M6, an M4 Pro or an M5 Max.