diff --git a/dm/gfx/nvn/dmsh_chain.h b/dm/gfx/nvn/dmsh_chain.h index 8a02bb5206..9ffa560aa5 100644 --- a/dm/gfx/nvn/dmsh_chain.h +++ b/dm/gfx/nvn/dmsh_chain.h @@ -1,3 +1,3 @@ // Generated file (dm/gfx/tools/dmsh_chain.py --header): the B shader library translation chain version this source tree expects (default options), one for every game; // the host loads only /b-/. Run dmsh_chain.py --header after changing the chain's tools and commit this file with the change -#define DMSH_CHAIN "39eb2f781d07" +#define DMSH_CHAIN "ffb778cb19be" diff --git a/dm/gfx/tests/ampere_orin/ampere_orin.sass b/dm/gfx/tests/ampere_orin/ampere_orin.sass new file mode 100644 index 0000000000..98a3ca86ac --- /dev/null +++ b/dm/gfx/tests/ampere_orin/ampere_orin.sass @@ -0,0 +1,9623 @@ +// Each op's program as the translator gets it (dm/gfx/tools/ampere_orin/capture.py): the GPU compiler's machine code with its global +// loads and stores made shared memory ones, disassembled by nvdisasm on the device. Thread t's words lie at shared memory word t * W +// (W = inputs + outputs): its inputs, then its outputs. + +// == fadd_rn +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_rn +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == ffma_rn +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fadd_rz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD.RZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_rz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL.RZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == ffma_rz +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.RZ R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fadd_rm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD.RM R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_rm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL.RM R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == ffma_rm +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.RM R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fadd_rp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD.RP R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_rp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL.RP R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == ffma_rp +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.RP R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fadd_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD.FTZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fadd_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD.SAT R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fsub +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD R7, R0, -R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL.FTZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMUL.SAT R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == ffma_ftz +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.FTZ R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == ffma_sat +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.SAT R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == ffma_ftz_sat +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R9, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ FFMA.FTZ.SAT R7, R0, R7, R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fadd_abs_neg +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FADD R7, |R0|, -R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmul_by_imm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FMUL R7, R2, 0.3333333432674407959 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == fmin +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMNMX R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmax +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMNMX R7, R0, R7, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmin_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMNMX.FTZ R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmax_nan +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMNMX.NAN R7, R0, R7, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fmin_nan +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FMNMX.NAN R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fcopysign +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ LOP3.LUT R7, R7, 0x80000000, R0, 0xb8, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fabs +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FADD R7, |R2|, -RZ ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == fneg +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FADD R7, -R2, -RZ ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == fsetp +// base register R24 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x38 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R24, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R24, R24, 0x40, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R24] ; + /*01e0*/ LDS R7, [R24+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FSETP.NEU.AND P3, PT, R0.reuse, R7.reuse, PT ; + /*0220*/ FSETP.NE.AND P0, PT, R0.reuse, R7.reuse, PT ; + /*0230*/ FSETP.GEU.AND P4, PT, R0.reuse, R7.reuse, PT ; + /*0240*/ FSETP.GTU.AND P5, PT, R0.reuse, R7.reuse, PT ; + /*0250*/ FSETP.GT.AND P1, PT, R0, R7, PT ; + /*0260*/ SEL R9, RZ, 0x1, P3 ; + /*0270*/ FSETP.GE.AND P2, PT, R0.reuse, R7.reuse, PT ; + /*0280*/ SEL R11, RZ, 0x1, !P0 ; + /*0290*/ STS [R24+0x8], R9 ; + /*02a0*/ SEL R3, RZ, 0x1, P4 ; + /*02b0*/ FSETP.NAN.AND P6, PT, R0, R7, PT ; + /*02c0*/ STS [R24+0xc], R11 ; + /*02d0*/ SEL R7, RZ, 0x1, P5 ; + /*02e0*/ SEL R13, RZ, 0x1, !P1 ; + /*02f0*/ STS [R24+0x10], R3 ; + /*0300*/ SEL R15, RZ, 0x1, !P2 ; + /*0310*/ SEL R17, RZ, 0x1, P0 ; + /*0320*/ STS [R24+0x14], R7 ; + /*0330*/ SEL R19, RZ, 0x1, !P3 ; + /*0340*/ SEL R21, RZ, 0x1, P2 ; + /*0350*/ STS [R24+0x18], R13 ; + /*0360*/ SEL R9, RZ, 0x1, !P4 ; + /*0370*/ SEL R11, RZ, 0x1, P6 ; + /*0380*/ STS [R24+0x1c], R15 ; + /*0390*/ SEL R3, RZ, 0x1, P1 ; + /*03a0*/ SEL R23, RZ, 0x1, !P6 ; + /*03b0*/ STS [R24+0x20], R17 ; + /*03c0*/ SEL R7, RZ, 0x1, !P5 ; + /*03d0*/ STS [R24+0x24], R19 ; + /*03e0*/ STS [R24+0x28], R21 ; + /*03f0*/ STS [R24+0x2c], R3 ; + /*0400*/ STS [R24+0x30], R7 ; + /*0410*/ STS [R24+0x34], R9 ; + /*0420*/ STS [R24+0x38], R11 ; + /*0430*/ STS [R24+0x3c], R23 ; + /*0440*/ EXIT ; + /*0450*/ BRA 0x450; + +// == fsetp_ftz +// base register R14 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x10 +// nop 0x0220 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R14, SR_TID.X ; + /*0010*/ IMAD.U32 R14, R14, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x8 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R14] ; + /*01e0*/ LDS R5, [R14+0x4] ; + /*01f0*/ NOP ; + /*0200*/ FSETP.NEU.FTZ.AND P2, PT, R0.reuse, R5.reuse, PT ; + /*0210*/ FSETP.GEU.FTZ.AND P3, PT, R0.reuse, R5.reuse, PT ; + /*0220*/ FSETP.GTU.FTZ.AND P0, PT, R0, R5, PT ; + /*0230*/ FSETP.NE.FTZ.AND P1, PT, R0, R5, PT ; + /*0240*/ NOP ; + /*0250*/ SEL R7, RZ, 0x1, P2 ; + /*0260*/ SEL R9, RZ, 0x1, P3 ; + /*0270*/ SEL R11, RZ, 0x1, !P0 ; + /*0280*/ STS [R14+0x8], R7 ; + /*0290*/ SEL R13, RZ, 0x1, !P1 ; + /*02a0*/ STS [R14+0xc], R9 ; + /*02b0*/ STS [R14+0x10], R11 ; + /*02c0*/ STS [R14+0x14], R13 ; + /*02d0*/ EXIT ; + /*02e0*/ BRA 0x2e0; + +// == fset_bf +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ FSET.BF.LT.AND R9, R0.reuse, R7.reuse, PT ; + /*0210*/ FSET.BF.NEU.AND R7, R0, R7, PT ; + /*0220*/ STS [R10+0x8], R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == fset_u32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R7, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ FSETP.GE.AND P0, PT, R0.reuse, R5.reuse, PT ; + /*0200*/ FSETP.NAN.AND P1, PT, R0, R5, PT ; + /*0210*/ NOP ; + /*0220*/ SEL R7, RZ, 0xffffffff, !P0 ; + /*0230*/ SEL R9, RZ, 0xffffffff, !P1 ; + /*0240*/ STS [R10+0x8], R7 ; + /*0250*/ STS [R10+0xc], R9 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; + +// == ftestp +// base register R14 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x4 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x14 +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R14, SR_TID.X ; + /*0010*/ IMAD.U32 R14, R14, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R14] ; + /*01e0*/ NOP ; + /*01f0*/ NOP ; + /*0200*/ LOP3.LUT R0, R2, 0x7fffffff, RZ, 0xc0, !PT ; + /*0210*/ ISETP.GE.U32.AND P4, PT, R0.reuse, 0x7f800000, PT ; + /*0220*/ ISETP.GE.U32.AND P0, PT, R0.reuse, 0x800000, PT ; + /*0230*/ ISETP.GE.U32.AND P1, PT, R0.reuse, 0x800000, !P4 ; + /*0240*/ ISETP.NE.U32.AND P3, PT, R0.reuse, 0x7f800000, PT ; + /*0250*/ ISETP.GT.U32.AND P2, PT, R0.reuse, 0x7f800000, PT ; + /*0260*/ ISETP.EQ.U32.OR P1, PT, R0, RZ, P1 ; + /*0270*/ ISETP.GT.U32.AND P0, PT, R0, RZ, !P0 ; + /*0280*/ SEL R3, RZ, 0x1, P4 ; + /*0290*/ SEL R7, RZ, 0x1, P3 ; + /*02a0*/ SEL R9, RZ, 0x1, P2 ; + /*02b0*/ STS [R14+0x4], R3 ; + /*02c0*/ SEL R11, RZ, 0x1, !P1 ; + /*02d0*/ SEL R13, RZ, 0x1, !P0 ; + /*02e0*/ STS [R14+0x8], R7 ; + /*02f0*/ STS [R14+0xc], R9 ; + /*0300*/ STS [R14+0x10], R11 ; + /*0310*/ STS [R14+0x14], R13 ; + /*0320*/ EXIT ; + /*0330*/ BRA 0x330; + +// == fsel_cmp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0200 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ FSETP.GT.AND P0, PT, R0, RZ, PT ; + /*0200*/ @!P0 LDS R7, [R8+0x8] ; + /*0210*/ NOP ; + /*0220*/ NOP ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == mufu_ex2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FSETP.GEU.AND P0, PT, R0, -126, PT ; + /*0200*/ @!P0 FMUL R0, R0, 0.5 ; + /*0210*/ MUFU.EX2 R7, R0 ; + /*0220*/ @!P0 FMUL R7, R7, R7 ; + /*0230*/ STS [R8+0x4], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == mufu_ex2_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.EX2 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_lg2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FSETP.GEU.AND P0, PT, |R0|, 1.175494350822287508e-38, PT ; + /*0200*/ @!P0 FMUL R0, R0, 16777216 ; + /*0210*/ MUFU.LG2 R7, R0 ; + /*0220*/ @!P0 FADD R7, R7, -24 ; + /*0230*/ STS [R8+0x4], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == mufu_lg2_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.LG2 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_rcp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ HFMA2.MMA R0, -RZ, RZ, 15, 0 ; + /*01f0*/ NOP ; + /*0200*/ FSETP.GEU.AND P1, PT, |R2|.reuse, 1.175494350822287508e-38, PT ; + /*0210*/ FSETP.GT.AND P0, PT, |R2|, 8.50705917302346158658e+37, PT ; + /*0220*/ FSEL R0, R0, 1, !P1 ; + /*0230*/ FSEL R7, R0, 0.25, !P0 ; + /*0240*/ FMUL R0, R2, R7 ; + /*0250*/ MUFU.RCP R0, R0 ; + /*0260*/ FMUL R7, R7, R0 ; + /*0270*/ STS [R8+0x4], R7 ; + /*0280*/ EXIT ; + /*0290*/ BRA 0x290; + +// == mufu_rcp_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.RCP R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_rsqrt +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FSETP.GEU.AND P0, PT, |R0|, 1.175494350822287508e-38, PT ; + /*0200*/ @!P0 FMUL R0, R0, 16777216 ; + /*0210*/ MUFU.RSQ R7, R0 ; + /*0220*/ @!P0 FMUL R7, R7, 4096 ; + /*0230*/ STS [R8+0x4], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == mufu_rsqrt_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.RSQ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_sqrt +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FSETP.GEU.AND P0, PT, |R0|, 1.175494350822287508e-38, PT ; + /*0200*/ @!P0 FMUL R0, R0, 16777216 ; + /*0210*/ MUFU.SQRT R7, R0 ; + /*0220*/ @!P0 FMUL R7, R7, 0.000244140625 ; + /*0230*/ STS [R8+0x4], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == mufu_sqrt_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.SQRT R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_sin +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FMUL.RZ R7, R2, 0.15915493667125701904 ; + /*0200*/ MUFU.SIN R7, R7 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == mufu_cos +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FMUL.RZ R7, R2, 0.15915493667125701904 ; + /*0200*/ MUFU.COS R7, R7 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == mufu_tanh +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.TANH R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == mufu_ex2_f16x2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.EX2.F16 R0, R2 ; + /*0200*/ MUFU.EX2.F16 R7, R2.H1 ; + /*0210*/ PRMT R7, R0, 0x5410, R7 ; + /*0220*/ STS [R8+0x4], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == mufu_tanh_f16x2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ MUFU.TANH.F16 R0, R2 ; + /*0200*/ MUFU.TANH.F16 R7, R2.H1 ; + /*0210*/ PRMT R7, R0, 0x5410, R7 ; + /*0220*/ STS [R8+0x4], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == fdiv_rn +// base register R22 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR36, c[0x0][0x118] +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R16, R5, c[0x0][0x160] +// nop 0x02b0 (feeds only the removed addressing): HFMA2.MMA R3, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x02c0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R16, R3, c[0x0][0x168] + /*0000*/ S2R R22, SR_TID.X ; + /*0010*/ IMAD.U32 R22, R22, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R16, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ LEA R16, R3, R16, 0x5 ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R22] ; + /*01e0*/ LDS R5, [R22+0x4] ; + /*01f0*/ BSSY B6, 0x2d0 ; + /*0200*/ MUFU.RCP R0, R5 ; + /*0210*/ FCHK P0, R4, R5 ; + /*0220*/ FFMA R7, -R5, R0, 1 ; + /*0230*/ FFMA R7, R0, R7, R0 ; + /*0240*/ FFMA R0, R4, R7, RZ ; + /*0250*/ FFMA R6, -R5, R0, R4 ; + /*0260*/ FFMA R0, R7, R6, R0 ; + /*0270*/ @!P0 BRA 0x2c0 ; + /*0280*/ MOV R20, 0x0 ; + /*0290*/ MOV R21, 0x0 ; + /*02a0*/ CALL.ABS.NOINC 0x0 ; + /*02b0*/ MOV R0, R4 ; + /*02c0*/ BSYNC B6 ; + /*02d0*/ NOP ; + /*02e0*/ NOP ; + /*02f0*/ STS [R22+0x8], R0 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == fdiv_full +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R6, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FSETP.GT.AND P0, PT, |R6|.reuse, 8.50705917302346158658e+37, PT ; + /*0220*/ FSETP.GEU.AND P1, PT, |R6|, 1.175494350822287508e-38, PT ; + /*0230*/ @P0 FMUL R6, R6, 0.25 ; + /*0240*/ @P0 FMUL R0, R0, 0.25 ; + /*0250*/ @!P1 FMUL R6, R6, 16777216 ; + /*0260*/ @!P1 FMUL R0, R0, 16777216 ; + /*0270*/ MUFU.RCP R7, R6 ; + /*0280*/ FMUL R7, R7, R0 ; + /*0290*/ STS [R8+0x8], R7 ; + /*02a0*/ EXIT ; + /*02b0*/ BRA 0x2b0; + +// == fdiv_approx +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R6, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ FSETP.GEU.AND P0, PT, |R6|, 1.175494350822287508e-38, PT ; + /*0220*/ @!P0 FMUL R6, R6, 16777216 ; + /*0230*/ @!P0 FMUL R0, R0, 16777216 ; + /*0240*/ MUFU.RCP R7, R6 ; + /*0250*/ FMUL R7, R7, R0 ; + /*0260*/ STS [R8+0x8], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == frcp_rn +// base register R22 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x4 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR36, c[0x0][0x118] +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R2, R5, c[0x0][0x160] +// nop 0x02a0 (feeds only the removed addressing): HFMA2.MMA R3, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x02b0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R2, R3, c[0x0][0x168] + /*0000*/ S2R R22, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R22, R22, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R2, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ IMAD R2, R3, 0x20, R2 ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R22] ; + /*01e0*/ BSSY B6, 0x2c0 ; + /*01f0*/ IADD3 R0, R4, 0x1800000, RZ ; + /*0200*/ LOP3.LUT R0, R0, 0x7f800000, RZ, 0xc0, !PT ; + /*0210*/ ISETP.GT.U32.AND P0, PT, R0, 0x1ffffff, PT ; + /*0220*/ @P0 BRA 0x270 ; + /*0230*/ MOV R20, 0x0 ; + /*0240*/ MOV R21, 0x0 ; + /*0250*/ CALL.ABS.NOINC 0x0 ; + /*0260*/ BRA 0x2b0 ; + /*0270*/ MUFU.RCP R3, R4 ; + /*0280*/ FFMA R0, R4, R3, -1 ; + /*0290*/ FADD.FTZ R0, -R0, -RZ ; + /*02a0*/ FFMA R4, R3, R0, R3 ; + /*02b0*/ BSYNC B6 ; + /*02c0*/ NOP ; + /*02d0*/ NOP ; + /*02e0*/ STS [R22+0x4], R4 ; + /*02f0*/ EXIT ; + /*0300*/ BRA 0x300; + +// == fsqrt_rn +// base register R22 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x4 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR36, c[0x0][0x118] +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R2, R5, c[0x0][0x160] +// nop 0x02a0 (feeds only the removed addressing): HFMA2.MMA R3, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x02b0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R2, R3, c[0x0][0x168] + /*0000*/ S2R R22, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R22, R22, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R2, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ LEA R2, R3, R2, 0x5 ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R22] ; + /*01e0*/ BSSY B6, 0x2c0 ; + /*01f0*/ IADD3 R0, R4, -0xd000000, RZ ; + /*0200*/ MUFU.RSQ R3, R4 ; + /*0210*/ ISETP.GT.U32.AND P0, PT, R0, 0x727fffff, PT ; + /*0220*/ @!P0 BRA 0x270 ; + /*0230*/ MOV R20, 0x0 ; + /*0240*/ MOV R21, 0x0 ; + /*0250*/ CALL.ABS.NOINC 0x0 ; + /*0260*/ BRA 0x2b0 ; + /*0270*/ FMUL.FTZ R5, R4, R3 ; + /*0280*/ FMUL.FTZ R3, R3, 0.5 ; + /*0290*/ FFMA R4, -R5, R5, R4 ; + /*02a0*/ FFMA R4, R4, R3, R5 ; + /*02b0*/ BSYNC B6 ; + /*02c0*/ NOP ; + /*02d0*/ NOP ; + /*02e0*/ STS [R22+0x4], R4 ; + /*02f0*/ EXIT ; + /*0300*/ BRA 0x300; + +// == f2i_s32_rni +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == frnd_rni +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FRND R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_s32_rzi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.TRUNC.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == frnd_rzi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FRND.TRUNC R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_s32_rmi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.FLOOR.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == frnd_rmi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FRND.FLOOR R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_s32_rpi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.CEIL.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == frnd_rpi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FRND.CEIL R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_u32_rzi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.U32.TRUNC.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_s32_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.FTZ.TRUNC.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_s16_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2I.S16.TRUNC.NTZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2i_u8_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2IP.U8.F32.NTZ R0, RZ, R2, RZ ; + /*0200*/ LOP3.LUT R7, R0, 0xff, RZ, 0xc0, !PT ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == f2i_s64 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R6, R3, R6, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R6, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): HFMA2.MMA R7, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R6, R7, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R6, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ NOP ; + /*0200*/ F2I.S64.TRUNC R4, R2 ; + /*0210*/ STS [R8+0x4], R4 ; + /*0220*/ STS [R8+0x8], R5 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == i2f_s32_rn +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2FP.F32.S32 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_s32_rz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2FP.F32.S32.RZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_s32_rm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2F.RM R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_s32_rp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2F.RP R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_u32_rn +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2FP.F32.U32 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_s16 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2F.S16 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == i2f_s64 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R6, R3, R6, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R6, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R7, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R6, R7, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R6, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R8] ; + /*01e0*/ LDS R5, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ I2F.S64 R5, R4 ; + /*0220*/ STS [R8+0x8], R5 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == f2f_f16_rn +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2F.F16.F32 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2f_f16_rz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ F2F.F16.F32.RZ R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == f2f_f32_from_f16 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R10] ; + /*01e0*/ NOP ; + /*01f0*/ NOP ; + /*0200*/ HADD2.F32 R7, -RZ, R2.reuse.H0_H0 ; + /*0210*/ HADD2.F32 R9, -RZ, R2.H1_H1 ; + /*0220*/ STS [R10+0x4], R7 ; + /*0230*/ STS [R10+0x8], R9 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == f2fp_pack_f16x2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ F2FP.PACK_AB R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == f2fp_pack_f16x2_relu +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ F2FP.RELU.PACK_AB R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == f2fp_pack_bf16x2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ F2FP.BF16.PACK_AB R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == f2f_tf32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ HFMA2.MMA R7, -RZ, RZ, 0, 0.00048828125 ; + /*01f0*/ NOP ; + /*0200*/ FSETP.GEU.AND P0, PT, |R0|, +INF , PT ; + /*0210*/ @!P0 IMAD.IADD.U32 R0, R0, 0x1, R7 ; + /*0220*/ LOP3.LUT R7, R0, 0xffffe000, RZ, 0xc0, !PT ; + /*0230*/ STS [R8+0x4], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == i2i_s8_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2I.S8.S32.SAT R0, R2 ; + /*0200*/ PRMT R7, R0, 0x8880, RZ ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == i2i_u16_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ I2I.U16.S32.SAT R0, R2 ; + /*0200*/ LOP3.LUT R7, R0, 0xffff, RZ, 0xc0, !PT ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == sext_s8 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ PRMT R7, R2, 0x8880, RZ ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == hadd2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HADD2 R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hadd2_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HADD2.FTZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hadd2_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HADD2.SAT R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hsub2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HADD2 R7, R0, -R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hmul2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMUL2 R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hmul2_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMUL2.FTZ R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hfma2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ HFMA2.MMA R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == hfma2_ftz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ HFMA2.MMA.FTZ R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == hfma2_sat +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ HFMA2.MMA.SAT R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == hfma2_relu +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ HFMA2.MMA.RELU R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == hmin2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMNMX2 R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hmax2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMNMX2 R7, R0, R7, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hmin2_nan +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMNMX2.NAN R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == hneg2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ HADD2 R7, -R2, -RZ.H0_H0 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == habs2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ HADD2 R7, |R2|, -RZ.H0_H0 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == hadd2_imm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ HADD2 R7, R2, 1, -1 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == hadd_f16 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R7, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ HADD2 R3, R0.reuse.H0_H0, R5.H0_H0 ; + /*0200*/ HMUL2 R6, R0.H1_H1, R5.H1_H1 ; + /*0210*/ NOP ; + /*0220*/ PRMT R0, R3, 0x7610, R0 ; + /*0230*/ LOP3.LUT R9, R6, 0xffff, RZ, 0xc0, !PT ; + /*0240*/ LOP3.LUT R7, R0, 0xffff, RZ, 0xc0, !PT ; + /*0250*/ STS [R10+0x8], R7 ; + /*0260*/ STS [R10+0xc], R9 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == hsetp2 +// base register R20 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x20 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R20, SR_TID.X ; + /*0010*/ IMAD.U32 R20, R20, 0x28, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R20] ; + /*01e0*/ LDS R7, [R20+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HSETP2.LT.AND P1, P2, R0.reuse, R7.reuse, PT ; + /*0220*/ HSETP2.EQ.AND P3, P0, R0, R7, PT ; + /*0230*/ SEL R9, RZ, 0x1, !P1 ; + /*0240*/ SEL R11, RZ, 0x1, !P2 ; + /*0250*/ HSETP2.GTU.AND P1, P2, R0.reuse, R7.reuse, PT ; + /*0260*/ STS [R20+0x8], R9 ; + /*0270*/ SEL R13, RZ, 0x1, !P3 ; + /*0280*/ HSETP2.NAN.AND P3, P4, R0, R7, PT ; + /*0290*/ STS [R20+0xc], R11 ; + /*02a0*/ SEL R3, RZ, 0x1, !P0 ; + /*02b0*/ SEL R7, RZ, 0x1, !P1 ; + /*02c0*/ STS [R20+0x10], R13 ; + /*02d0*/ SEL R15, RZ, 0x1, !P2 ; + /*02e0*/ SEL R17, RZ, 0x1, !P3 ; + /*02f0*/ STS [R20+0x14], R3 ; + /*0300*/ SEL R19, RZ, 0x1, !P4 ; + /*0310*/ STS [R20+0x18], R7 ; + /*0320*/ STS [R20+0x1c], R15 ; + /*0330*/ STS [R20+0x20], R17 ; + /*0340*/ STS [R20+0x24], R19 ; + /*0350*/ EXIT ; + /*0360*/ BRA 0x360; + +// == hset2 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ HSET2.BF.LT.AND R9, R0.reuse, R7.reuse, PT ; + /*0210*/ HSET2.GEU.AND R7, R0, R7, PT ; + /*0220*/ STS [R10+0x8], R9 ; + /*0230*/ STS [R10+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == bfma2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ HFMA2.BF16_V2 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == bmin2 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMNMX2.BF16_V2 R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == bmax2_nan +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ HMNMX2.BF16_V2.NAN R7, R0, R7, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == iadd +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IADD3 R7, R0, R7, RZ ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == iadd3 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IADD3 R7, -R6, R0, R7 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == iadd_sat +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R9, RZ, RZ, 0x4 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x8 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ IADD3 R6, R0, R5, RZ ; + /*0210*/ PLOP3.LUT P0, PT, R0.SIGN, R5.SIGN, R6.SIGN, 0x2, 0x0 ; + /*0220*/ PLOP3.LUT P1, PT, R0.SIGN, R5.SIGN, R6.SIGN, 0x40, 0x0 ; + /*0230*/ NOP ; + /*0240*/ SEL R7, R6, 0x7fffffff, !P0 ; + /*0250*/ SEL R7, R7, 0x80000000, !P1 ; + /*0260*/ STS [R10+0x8], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == isub_sat +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R9, RZ, RZ, 0x4 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x8 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ IADD3 R6, R0, -R5, RZ ; + /*0210*/ PLOP3.LUT P0, PT, R0.SIGN, R5.SIGN, R6.SIGN, 0x8, 0x0 ; + /*0220*/ PLOP3.LUT P1, PT, R0.SIGN, R5.SIGN, R6.SIGN, 0x10, 0x0 ; + /*0230*/ NOP ; + /*0240*/ SEL R7, R6, 0x7fffffff, !P0 ; + /*0250*/ SEL R7, R7, 0x80000000, !P1 ; + /*0260*/ STS [R10+0x8], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == iadd64 +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R11, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R11, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 9.5367431640625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R12] ; + /*01e0*/ LDS R6, [R12+0x4] ; + /*01f0*/ LDS R5, [R12+0x8] ; + /*0200*/ LDS R7, [R12+0xc] ; + /*0210*/ NOP ; + /*0220*/ IADD3 R9, P0, R0, R5, RZ ; + /*0230*/ NOP ; + /*0240*/ IADD3.X R7, R6, R7, RZ, P0, !PT ; + /*0250*/ STS [R12+0x10], R9 ; + /*0260*/ STS [R12+0x14], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == isub64 +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R11, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R11, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 9.5367431640625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R12] ; + /*01e0*/ LDS R6, [R12+0x4] ; + /*01f0*/ LDS R5, [R12+0x8] ; + /*0200*/ LDS R7, [R12+0xc] ; + /*0210*/ NOP ; + /*0220*/ IADD3 R9, P0, R0, -R5, RZ ; + /*0230*/ NOP ; + /*0240*/ IADD3.X R7, R6, ~R7, RZ, P0, !PT ; + /*0250*/ STS [R12+0x10], R9 ; + /*0260*/ STS [R12+0x14], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == imul_lo +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IMAD R7, R0, R7, RZ ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imul_hi_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IMAD.HI.U32 R7, R0, R7, RZ ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imul_hi_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IMAD.HI R7, R0, R7, RZ ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imul_wide_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R7, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R6, R3, R6, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R6, R7, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R6, R7, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R6, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R8] ; + /*01e0*/ LDS R5, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ IMAD.WIDE R4, R4, R5, RZ ; + /*0210*/ STS [R8+0x8], R4 ; + /*0220*/ STS [R8+0xc], R5 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imul_wide_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R7, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R6, R3, R6, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R6, R7, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R6, R7, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R6, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R8] ; + /*01e0*/ LDS R5, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ IMAD.WIDE.U32 R4, R4, R5, RZ ; + /*0210*/ STS [R8+0x8], R4 ; + /*0220*/ STS [R8+0xc], R5 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imad_lo +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IMAD R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == imad_hi_s32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0200 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R9, [R10] ; + /*01e0*/ LDS R0, [R10+0x4] ; + /*01f0*/ LDS R7, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ MOV R6, RZ ; + /*0220*/ NOP ; + /*0230*/ IMAD.HI R9, R9, R0, R6 ; + /*0240*/ STS [R10+0xc], R9 ; + /*0250*/ EXIT ; + /*0260*/ BRA 0x260; + +// == imul64 +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R11, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0230 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R11, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 9.5367431640625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R12] ; + /*01e0*/ LDS R5, [R12+0x4] ; + /*01f0*/ LDS R6, [R12+0x8] ; + /*0200*/ LDS R9, [R12+0xc] ; + /*0210*/ NOP ; + /*0220*/ IMAD R5, R5, R6, RZ ; + /*0230*/ IMAD.WIDE.U32 R6, R0, R6, RZ ; + /*0240*/ IMAD R9, R0, R9, R5 ; + /*0250*/ NOP ; + /*0260*/ IADD3 R7, R7, R9, RZ ; + /*0270*/ STS [R12+0x10], R6 ; + /*0280*/ STS [R12+0x14], R7 ; + /*0290*/ EXIT ; + /*02a0*/ BRA 0x2a0; + +// == imul24 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0200 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ SGXT R0, R0, 0x18 ; + /*0210*/ SGXT R7, R5, 0x18 ; + /*0220*/ NOP ; + /*0230*/ IMAD R7, R0, R7, RZ ; + /*0240*/ STS [R10+0x8], R7 ; + /*0250*/ EXIT ; + /*0260*/ BRA 0x260; + +// == imin_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IMNMX R7, R0, R7, PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imax_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ IMNMX.U32 R7, R0, R7, !PT ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == imin_s64 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x10 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0200 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R6, [R10+0x4] ; + /*01f0*/ LDS R7, [R10+0x8] ; + /*0200*/ LDS R9, [R10+0xc] ; + /*0210*/ NOP ; + /*0220*/ NOP ; + /*0230*/ ISETP.LT.U32.AND P0, PT, R0, R7, PT ; + /*0240*/ ISETP.LT.AND.EX P0, PT, R6, R9, PT, P0 ; + /*0250*/ SEL R7, R0, R7, P0 ; + /*0260*/ SEL R9, R6, R9, P0 ; + /*0270*/ STS [R10+0x10], R7 ; + /*0280*/ STS [R10+0x14], R9 ; + /*0290*/ EXIT ; + /*02a0*/ BRA 0x2a0; + +// == iabs +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ IABS R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == ineg +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ IADD3 R7, -R2, RZ, RZ ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == isad +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ VABSDIFF.U32 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == idiv_u32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R0, R3, R0, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R3, RZ, RZ, 0x4 +// nop 0x02a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R3, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R6, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ I2F.U32.RP R8, R7 ; + /*0210*/ IMAD.MOV R9, RZ, RZ, -R7 ; + /*0220*/ ISETP.NE.U32.AND P2, PT, R7, RZ, PT ; + /*0230*/ MUFU.RCP R8, R8 ; + /*0240*/ IADD3 R4, R8, 0xffffffe, RZ ; + /*0250*/ F2I.FTZ.U32.TRUNC.NTZ R5, R4 ; + /*0260*/ HFMA2.MMA R4, -RZ, RZ, 0, 0 ; + /*0270*/ IMAD R9, R9, R5, RZ ; + /*0280*/ IMAD.HI.U32 R5, R5, R9, R4 ; + /*0290*/ IMAD.HI.U32 R5, R5, R6, RZ ; + /*02a0*/ IMAD.MOV R2, RZ, RZ, -R5 ; + /*02b0*/ IMAD R6, R7, R2, R6 ; + /*02c0*/ NOP ; + /*02d0*/ ISETP.GE.U32.AND P0, PT, R6, R7, PT ; + /*02e0*/ @P0 IADD3 R6, -R7, R6, RZ ; + /*02f0*/ @P0 IADD3 R5, R5, 0x1, RZ ; + /*0300*/ ISETP.GE.U32.AND P1, PT, R6, R7, PT ; + /*0310*/ @P1 IADD3 R5, R5, 0x1, RZ ; + /*0320*/ @!P2 LOP3.LUT R5, RZ, R7, RZ, 0x33, !PT ; + /*0330*/ STS [R10+0x8], R5 ; + /*0340*/ EXIT ; + /*0350*/ BRA 0x350; + +// == irem_u32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R0, R3, 0x20, R0 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R3, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R3, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R6, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ I2F.U32.RP R8, R7 ; + /*0220*/ IADD3 R9, RZ, -R7, RZ ; + /*0230*/ ISETP.NE.U32.AND P1, PT, R7, RZ, PT ; + /*0240*/ MUFU.RCP R8, R8 ; + /*0250*/ IADD3 R4, R8, 0xffffffe, RZ ; + /*0260*/ F2I.FTZ.U32.TRUNC.NTZ R5, R4 ; + /*0270*/ IMAD.MOV.U32 R4, RZ, RZ, RZ ; + /*0280*/ IMAD R9, R9, R5, RZ ; + /*0290*/ IMAD.HI.U32 R5, R5, R9, R4 ; + /*02a0*/ IMAD.HI.U32 R5, R5, R6, RZ ; + /*02b0*/ IADD3 R5, -R5, RZ, RZ ; + /*02c0*/ IMAD R6, R7, R5, R6 ; + /*02d0*/ ISETP.GE.U32.AND P0, PT, R6, R7, PT ; + /*02e0*/ @P0 IMAD.IADD R6, R6, 0x1, -R7 ; + /*02f0*/ ISETP.GE.U32.AND P0, PT, R6, R7, PT ; + /*0300*/ @P0 IADD3 R6, -R7, R6, RZ ; + /*0310*/ @!P1 LOP3.LUT R6, RZ, R7, RZ, 0x33, !PT ; + /*0320*/ STS [R10+0x8], R6 ; + /*0330*/ EXIT ; + /*0340*/ BRA 0x340; + +// == idiv_s32 +// base register R13 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R0, R3, 0x20, R0 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x02e0 (feeds only the removed addressing): IMAD.MOV.U32 R3, RZ, RZ, 0x4 +// nop 0x0330 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R3, c[0x0][0x168] + /*0000*/ S2R R13, SR_TID.X ; + /*0010*/ IMAD.U32 R13, R13, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R6, [R13] ; + /*01e0*/ LDS R7, [R13+0x4] ; + /*01f0*/ IABS R12, R6 ; + /*0200*/ IABS R9, R7.reuse ; + /*0210*/ IABS R3, R7 ; + /*0220*/ I2F.RP R8, R9 ; + /*0230*/ MOV R2, R12 ; + /*0240*/ IMAD.MOV R3, RZ, RZ, -R3 ; + /*0250*/ LOP3.LUT R6, R6, R7, RZ, 0x3c, !PT ; + /*0260*/ ISETP.GE.AND P1, PT, R6, RZ, PT ; + /*0270*/ MUFU.RCP R8, R8 ; + /*0280*/ IADD3 R4, R8, 0xffffffe, RZ ; + /*0290*/ F2I.FTZ.U32.TRUNC.NTZ R5, R4 ; + /*02a0*/ IMAD.MOV.U32 R4, RZ, RZ, RZ ; + /*02b0*/ IMAD.MOV R10, RZ, RZ, -R5 ; + /*02c0*/ IMAD R11, R10, R9, RZ ; + /*02d0*/ IMAD.HI.U32 R5, R5, R11, R4 ; + /*02e0*/ IMAD.HI.U32 R5, R5, R2, RZ ; + /*02f0*/ IMAD R2, R5, R3, R2 ; + /*0300*/ NOP ; + /*0310*/ ISETP.GT.U32.AND P2, PT, R9, R2, PT ; + /*0320*/ @!P2 IADD3 R2, R2, -R9.reuse, RZ ; + /*0330*/ @!P2 IADD3 R5, R5, 0x1, RZ ; + /*0340*/ ISETP.GE.U32.AND P0, PT, R2, R9, PT ; + /*0350*/ NOP ; + /*0360*/ ISETP.NE.AND P2, PT, R7, RZ, PT ; + /*0370*/ @P0 IADD3 R5, R5, 0x1, RZ ; + /*0380*/ @!P1 IADD3 R5, -R5, RZ, RZ ; + /*0390*/ @!P2 LOP3.LUT R5, RZ, R7, RZ, 0x33, !PT ; + /*03a0*/ STS [R13+0x8], R5 ; + /*03b0*/ EXIT ; + /*03c0*/ BRA 0x3c0; + +// == irem_s32 +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R0, R3, 0x20, R0 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x02d0 (feeds only the removed addressing): MOV R3, 0x4 +// nop 0x02f0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R3, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R11, [R12] ; + /*01e0*/ LDS R6, [R12+0x4] ; + /*01f0*/ IABS R10, R11 ; + /*0200*/ ISETP.GE.AND P2, PT, R11, RZ, PT ; + /*0210*/ IABS R8, R6.reuse ; + /*0220*/ IMAD.MOV.U32 R2, RZ, RZ, R10 ; + /*0230*/ IABS R3, R6 ; + /*0240*/ I2F.RP R7, R8 ; + /*0250*/ IMAD.MOV R3, RZ, RZ, -R3 ; + /*0260*/ MUFU.RCP R7, R7 ; + /*0270*/ IADD3 R4, R7, 0xffffffe, RZ ; + /*0280*/ F2I.FTZ.U32.TRUNC.NTZ R5, R4 ; + /*0290*/ HFMA2.MMA R4, -RZ, RZ, 0, 0 ; + /*02a0*/ IADD3 R9, RZ, -R5, RZ ; + /*02b0*/ IMAD R9, R9, R8, RZ ; + /*02c0*/ IMAD.HI.U32 R5, R5, R9, R4 ; + /*02d0*/ IMAD.HI.U32 R5, R5, R2, RZ ; + /*02e0*/ IMAD R5, R5, R3, R2 ; + /*02f0*/ NOP ; + /*0300*/ ISETP.GT.U32.AND P0, PT, R8, R5, PT ; + /*0310*/ NOP ; + /*0320*/ @!P0 IADD3 R5, R5, -R8, RZ ; + /*0330*/ ISETP.NE.AND P0, PT, R6, RZ, PT ; + /*0340*/ ISETP.GT.U32.AND P1, PT, R8, R5, PT ; + /*0350*/ @!P1 IMAD.IADD R5, R5, 0x1, -R8 ; + /*0360*/ @!P2 IMAD.MOV R5, RZ, RZ, -R5 ; + /*0370*/ @!P0 LOP3.LUT R5, RZ, R6, RZ, 0x33, !PT ; + /*0380*/ STS [R12+0x8], R5 ; + /*0390*/ EXIT ; + /*03a0*/ BRA 0x3a0; + +// == idiv_u32_by_7 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R0, R3, R0, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R9, c[0x0][0x160] +// nop 0x0250 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R10] ; + /*01e0*/ I2F.U32.RP R6, 0x7 ; + /*01f0*/ MUFU.RCP R6, R6 ; + /*0200*/ IADD3 R4, R6, 0xffffffe, RZ ; + /*0210*/ F2I.FTZ.U32.TRUNC.NTZ R5, R4 ; + /*0220*/ HFMA2.MMA R4, -RZ, RZ, 0, 0 ; + /*0230*/ IMAD R7, R5, -0x7, RZ ; + /*0240*/ IMAD.HI.U32 R5, R5, R7, R4 ; + /*0250*/ IMAD.HI.U32 R5, R5, R2, RZ ; + /*0260*/ IMAD R7, R5, -0x7, R2 ; + /*0270*/ NOP ; + /*0280*/ ISETP.GE.U32.AND P0, PT, R7, 0x7, PT ; + /*0290*/ @P0 IADD3 R7, R7, -0x7, RZ ; + /*02a0*/ @P0 IADD3 R5, R5, 0x1, RZ ; + /*02b0*/ ISETP.GE.U32.AND P1, PT, R7, 0x7, PT ; + /*02c0*/ @P1 IADD3 R5, R5, 0x1, RZ ; + /*02d0*/ STS [R10+0x4], R5 ; + /*02e0*/ EXIT ; + /*02f0*/ BRA 0x2f0; + +// == isetp +// base register R16 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x18 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R16, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R16, R16, 0x20, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R16] ; + /*01e0*/ LDS R7, [R16+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ ISETP.GE.AND P1, PT, R0.reuse, R7.reuse, PT ; + /*0220*/ ISETP.GE.U32.AND P0, PT, R0.reuse, R7.reuse, PT ; + /*0230*/ SEL R9, RZ, 0x1, P1 ; + /*0240*/ ISETP.GE.AND P1, PT, R0.reuse, R7.reuse, PT ; + /*0250*/ ISETP.GT.U32.AND P2, PT, R0.reuse, R7.reuse, PT ; + /*0260*/ STS [R16+0x8], R9 ; + /*0270*/ ISETP.NE.AND P4, PT, R0, R7, PT ; + /*0280*/ ISETP.NE.U32.AND P3, PT, R0, R7, PT ; + /*0290*/ SEL R3, RZ, 0x1, P0 ; + /*02a0*/ SEL R7, RZ, 0x1, !P1 ; + /*02b0*/ SEL R11, RZ, 0x1, !P2 ; + /*02c0*/ STS [R16+0xc], R3 ; + /*02d0*/ SEL R13, RZ, 0x1, P4 ; + /*02e0*/ SEL R15, RZ, 0x1, !P3 ; + /*02f0*/ STS [R16+0x10], R7 ; + /*0300*/ STS [R16+0x14], R11 ; + /*0310*/ STS [R16+0x18], R13 ; + /*0320*/ STS [R16+0x1c], R15 ; + /*0330*/ EXIT ; + /*0340*/ BRA 0x340; + +// == isetp_and +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0200 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x8 +// nop 0x0230 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x10 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ LDS R7, [R10+0xc] ; + /*0210*/ ISETP.GE.AND P0, PT, R6, R7, PT ; + /*0220*/ NOP ; + /*0230*/ ISETP.LT.AND P1, PT, R0.reuse, R5.reuse, !P0 ; + /*0240*/ ISETP.GE.U32.OR P0, PT, R0, R5, !P0 ; + /*0250*/ NOP ; + /*0260*/ SEL R7, RZ, 0x1, !P1 ; + /*0270*/ SEL R9, RZ, 0x1, !P0 ; + /*0280*/ STS [R10+0x10], R7 ; + /*0290*/ STS [R10+0x14], R9 ; + /*02a0*/ EXIT ; + /*02b0*/ BRA 0x2b0; + +// == isetp64 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0200 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x8 +// nop 0x0230 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x10 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R7, [R10+0x8] ; + /*0200*/ LDS R6, [R10+0xc] ; + /*0210*/ ISETP.GE.U32.AND P0, PT, R0, R7, PT ; + /*0220*/ NOP ; + /*0230*/ ISETP.GE.AND.EX P1, PT, R5.reuse, R6.reuse, PT, P0 ; + /*0240*/ ISETP.GE.U32.AND.EX P0, PT, R5, R6, PT, P0 ; + /*0250*/ NOP ; + /*0260*/ SEL R7, RZ, 0x1, P1 ; + /*0270*/ SEL R9, RZ, 0x1, P0 ; + /*0280*/ STS [R10+0x10], R7 ; + /*0290*/ STS [R10+0x14], R9 ; + /*02a0*/ EXIT ; + /*02b0*/ BRA 0x2b0; + +// == plop3 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x4 +// nop 0x0220 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0xc ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R6, [R8] ; + /*01e0*/ LDS R0, [R8+0x4] ; + /*01f0*/ LDS R5, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ ISETP.NE.U32.AND P0, PT, R0, RZ, PT ; + /*0220*/ ISETP.NE.U32.XOR P0, PT, R6, RZ, P0 ; + /*0230*/ ISETP.NE.U32.OR P0, PT, R5, RZ, P0 ; + /*0240*/ NOP ; + /*0250*/ SEL R7, RZ, 0x1, P0 ; + /*0260*/ STS [R8+0xc], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == shl +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHF.L.U32 R7, R0, R7, RZ ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == shr_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHF.R.U32.HI R7, RZ, R7, R0 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == shr_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHF.R.S32.HI R7, RZ, R7, R0 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == shl_imm +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R10] ; + /*01e0*/ NOP ; + /*01f0*/ NOP ; + /*0200*/ SHF.L.U32 R7, R2, 0x7, RZ ; + /*0210*/ SHF.R.S32.HI R9, RZ, 0xd, R2 ; + /*0220*/ STS [R10+0x4], R7 ; + /*0230*/ STS [R10+0x8], R9 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shf_l_wrap +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.L.W.U32.HI R7, R0, R6, R7 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shf_l_clamp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.L.U32.HI R7, R0, R6, R7 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shf_r_wrap +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.R.W.U32 R7, R0, R6, R7 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shf_r_clamp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.R.U32 R7, R0, R6, R7 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shl64 +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R12] ; + /*01e0*/ LDS R7, [R12+0x4] ; + /*01f0*/ LDS R9, [R12+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.L.U32 R11, R0.reuse, R9.reuse, RZ ; + /*0230*/ SHF.L.U64.HI R7, R0, R9, R7 ; + /*0240*/ STS [R12+0xc], R11 ; + /*0250*/ STS [R12+0x10], R7 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; + +// == shr64_u +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.R.U64 R9, R0, R6.reuse, R7.reuse ; + /*0230*/ SHF.R.U32.HI R7, RZ, R6, R7 ; + /*0240*/ STS [R10+0xc], R9 ; + /*0250*/ STS [R10+0x10], R7 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; + +// == shr64_s +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ SHF.R.S64 R9, R0, R6.reuse, R7.reuse ; + /*0230*/ SHF.R.S32.HI R7, RZ, R6, R7 ; + /*0240*/ STS [R10+0xc], R9 ; + /*0250*/ STS [R10+0x10], R7 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; + +// == lop3 +// base register R24 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0xc +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x28 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R24, SR_TID.X ; + /*0010*/ IMAD.U32 R24, R24, 0x34, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R24] ; + /*01e0*/ LDS R7, [R24+0x4] ; + /*01f0*/ LDS R6, [R24+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ LOP3.LUT R9, R0.reuse, R7.reuse, R6.reuse, 0x96, !PT ; + /*0230*/ LOP3.LUT R11, R0.reuse, R7.reuse, R6.reuse, 0xe8, !PT ; + /*0240*/ LOP3.LUT R13, R0.reuse, R7.reuse, R6.reuse, 0xca, !PT ; + /*0250*/ STS [R24+0xc], R9 ; + /*0260*/ LOP3.LUT R15, R0.reuse, R7.reuse, R6.reuse, 0x1, !PT ; + /*0270*/ LOP3.LUT R17, R0.reuse, R7.reuse, R6.reuse, 0x80, !PT ; + /*0280*/ STS [R24+0x10], R11 ; + /*0290*/ LOP3.LUT R19, R0, R7, R6, 0x6a, !PT ; + /*02a0*/ LOP3.LUT R21, R0.reuse, R7.reuse, R6.reuse, 0xf8, !PT ; + /*02b0*/ STS [R24+0x14], R13 ; + /*02c0*/ LOP3.LUT R23, R0.reuse, R7.reuse, R6.reuse, 0x1e, !PT ; + /*02d0*/ LOP3.LUT R3, R0.reuse, R7.reuse, R6.reuse, 0xb4, !PT ; + /*02e0*/ STS [R24+0x18], R15 ; + /*02f0*/ LOP3.LUT R7, R0, R7, R6, 0x3c, !PT ; + /*0300*/ STS [R24+0x1c], R17 ; + /*0310*/ STS [R24+0x20], R19 ; + /*0320*/ STS [R24+0x24], R21 ; + /*0330*/ STS [R24+0x28], R23 ; + /*0340*/ STS [R24+0x2c], R3 ; + /*0350*/ STS [R24+0x30], R7 ; + /*0360*/ EXIT ; + /*0370*/ BRA 0x370; + +// == logic +// base register R14 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x10 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R14, SR_TID.X ; + /*0010*/ IMAD.U32 R14, R14, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R14] ; + /*01e0*/ LDS R7, [R14+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ LOP3.LUT R9, R0.reuse, R7.reuse, RZ, 0xc0, !PT ; + /*0220*/ LOP3.LUT R11, R0.reuse, R7.reuse, RZ, 0xfc, !PT ; + /*0230*/ LOP3.LUT R13, R0.reuse, R7.reuse, RZ, 0x3c, !PT ; + /*0240*/ STS [R14+0x8], R9 ; + /*0250*/ LOP3.LUT R7, R0, R7, RZ, 0xc, !PT ; + /*0260*/ STS [R14+0xc], R11 ; + /*0270*/ STS [R14+0x10], R13 ; + /*0280*/ STS [R14+0x14], R7 ; + /*0290*/ EXIT ; + /*02a0*/ BRA 0x2a0; + +// == prmt +// base register R20 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0xc +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x1c +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R20, SR_TID.X ; + /*0010*/ IMAD.U32 R20, R20, 0x28, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R20] ; + /*01e0*/ LDS R7, [R20+0x4] ; + /*01f0*/ LDS R6, [R20+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ PRMT R9, R0.reuse, R6.reuse, R7.reuse ; + /*0230*/ PRMT.F4E R11, R0.reuse, R6.reuse, R7.reuse ; + /*0240*/ PRMT.B4E R13, R0.reuse, R6.reuse, R7.reuse ; + /*0250*/ STS [R20+0xc], R9 ; + /*0260*/ PRMT.RC8 R15, R0.reuse, R6.reuse, R7.reuse ; + /*0270*/ PRMT.ECL R17, R0.reuse, R6.reuse, R7.reuse ; + /*0280*/ STS [R20+0x10], R11 ; + /*0290*/ PRMT.ECR R19, R0, R6, R7 ; + /*02a0*/ PRMT.RC16 R7, R0, R6, R7 ; + /*02b0*/ STS [R20+0x14], R13 ; + /*02c0*/ STS [R20+0x18], R15 ; + /*02d0*/ STS [R20+0x1c], R17 ; + /*02e0*/ STS [R20+0x20], R19 ; + /*02f0*/ STS [R20+0x24], R7 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == prmt_imm +// base register R12 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x8 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0xc +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R12, SR_TID.X ; + /*0010*/ IMAD.U32 R12, R12, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R12] ; + /*01e0*/ LDS R7, [R12+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ PRMT R9, R0.reuse, 0x3210, R7.reuse ; + /*0220*/ PRMT R11, R0.reuse, 0x5140, R7.reuse ; + /*0230*/ PRMT R7, R0, 0x8880, R7 ; + /*0240*/ STS [R12+0x8], R9 ; + /*0250*/ STS [R12+0xc], R11 ; + /*0260*/ STS [R12+0x10], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == bfe_u32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x4 +// nop 0x0220 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0xc ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ PRMT R5, R6, 0x7604, R5 ; + /*0220*/ PRMT R9, RZ, 0x4, R5.reuse ; + /*0230*/ PRMT R6, RZ, 0x5, R5 ; + /*0240*/ NOP ; + /*0250*/ SHF.R.U32.HI R9, RZ, R9, R0 ; + /*0260*/ SGXT.U32 R7, R9, R6 ; + /*0270*/ STS [R10+0xc], R7 ; + /*0280*/ EXIT ; + /*0290*/ BRA 0x290; + +// == bfe_s32 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x4 +// nop 0x0220 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0xc ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ NOP ; + /*0210*/ PRMT R5, R6, 0x7604, R5 ; + /*0220*/ PRMT R9, RZ, 0x4, R5.reuse ; + /*0230*/ PRMT R6, RZ, 0x5, R5 ; + /*0240*/ NOP ; + /*0250*/ SHF.R.S32.HI R9, RZ, R9, R0 ; + /*0260*/ SGXT R7, R9, R6 ; + /*0270*/ STS [R10+0xc], R7 ; + /*0280*/ EXIT ; + /*0290*/ BRA 0x290; + +// == bfi +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0220 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x4 +// nop 0x0240 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x10 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R6, [R10+0x4] ; + /*01f0*/ LDS R5, [R10+0x8] ; + /*0200*/ LDS R8, [R10+0xc] ; + /*0210*/ PRMT R5, R8, 0x7604, R5 ; + /*0220*/ PRMT R7, RZ, 0x4, R5.reuse ; + /*0230*/ PRMT R8, RZ, 0x5, R5 ; + /*0240*/ NOP ; + /*0250*/ BMSK R9, R7, R8 ; + /*0260*/ NOP ; + /*0270*/ SHF.L.U32 R7, R0, R7, RZ ; + /*0280*/ LOP3.LUT R7, R7, R9, R6, 0xe2, !PT ; + /*0290*/ STS [R10+0x10], R7 ; + /*02a0*/ EXIT ; + /*02b0*/ BRA 0x2b0; + +// == bmsk_clamp +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ BMSK R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == bmsk_wrap +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ BMSK.W R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == szext_s +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SGXT R7, R0, R7 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == popc +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ POPC R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == popc64 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R6, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ POPC R0, R0 ; + /*0220*/ POPC R7, R6 ; + /*0230*/ IADD3 R7, R0, R7, RZ ; + /*0240*/ STS [R8+0x8], R7 ; + /*0250*/ EXIT ; + /*0260*/ BRA 0x260; + +// == clz +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FLO.U32 R0, R2 ; + /*0200*/ IADD3 R7, -R0, 0x1f, RZ ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == clz64 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0x8 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R5, [R8+0x4] ; + /*01f0*/ ISETP.NE.U32.AND P0, PT, R5, RZ, PT ; + /*0200*/ SEL R0, R0, R5, !P0 ; + /*0210*/ NOP ; + /*0220*/ FLO.U32 R0, R0 ; + /*0230*/ NOP ; + /*0240*/ IADD3 R6, -R0.reuse, 0x1f, RZ ; + /*0250*/ IADD3 R7, -R0, 0x3f, RZ ; + /*0260*/ @P0 IMAD.MOV R7, RZ, RZ, R6 ; + /*0270*/ STS [R8+0x8], R7 ; + /*0280*/ EXIT ; + /*0290*/ BRA 0x290; + +// == bfind_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FLO.U32 R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == bfind_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FLO R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == bfind_shiftamt +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ FLO.U32.SH R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == brev +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ BREV R7, R2 ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == fns +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x04d0 (feeds only the removed addressing): IMAD.MOV.U32 R7, RZ, RZ, 0x4 +// nop 0x0520 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R7, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ IMAD.MOV.U32 R5, RZ, RZ, 0xc ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ IMAD R0, R3, 0x20, R0 ; + /*01c0*/ NOP ; + /*01d0*/ LDS R5, [R10] ; + /*01e0*/ LDS R6, [R10+0x4] ; + /*01f0*/ LDS R4, [R10+0x8] ; + /*0200*/ BSSY B0, 0x310 ; + /*0210*/ PLOP3.LUT P0, PT, PT, PT, PT, 0x8, 0x0 ; + /*0220*/ LOP3.LUT R4, R4, 0x1f, RZ, 0xc0, !PT ; + /*0230*/ ISETP.EQ.AND P1, PT, R4, RZ, PT ; + /*0240*/ @P1 BRA 0x2c0 ; + /*0250*/ ISETP.GT.AND P1, PT, R4, RZ, PT ; + /*0260*/ @P1 BRA 0x300 ; + /*0270*/ BREV R5, R5 ; + /*0280*/ PLOP3.LUT P0, PT, PT, PT, PT, 0x80, 0x0 ; + /*0290*/ IMAD.MOV R4, RZ, RZ, -R4 ; + /*02a0*/ IADD3 R6, -R6, 0x1f, RZ ; + /*02b0*/ BRA 0x300 ; + /*02c0*/ IMAD.MOV.U32 R3, RZ, RZ, 0x1 ; + /*02d0*/ IMAD.MOV.U32 R4, RZ, RZ, 0x1 ; + /*02e0*/ SHF.L.U32 R2, R3, R6, RZ ; + /*02f0*/ LOP3.LUT R5, R5, R2, RZ, 0xc0, !PT ; + /*0300*/ BSYNC B0 ; + /*0310*/ IMAD.MOV.U32 R3, RZ, RZ, -0x1 ; + /*0320*/ SHF.L.U32 R6, R3, R6, RZ ; + /*0330*/ LOP3.LUT R5, R5, R6, RZ, 0xc0, !PT ; + /*0340*/ PRMT R2, R5, 0x7710, RZ ; + /*0350*/ POPC R3, R5 ; + /*0360*/ POPC R7, R2 ; + /*0370*/ ISETP.LT.U32.AND P4, PT, R7, R4, PT ; + /*0380*/ IMAD.IADD R7, R4, 0x1, -R7 ; + /*0390*/ SEL R7, R7, R4, P4 ; + /*03a0*/ @P4 SHF.R.U32.HI R5, RZ, 0x10, R5 ; + /*03b0*/ PRMT R6, R5, 0x7770, RZ ; + /*03c0*/ POPC R6, R6 ; + /*03d0*/ ISETP.LT.U32.AND P3, PT, R6, R7, PT ; + /*03e0*/ @P3 SHF.R.U32.HI R5, RZ, 0x8, R5 ; + /*03f0*/ @P3 IMAD.IADD R7, R7, 0x1, -R6 ; + /*0400*/ SGXT.U32 R8, R5, 0x4 ; + /*0410*/ POPC R8, R8 ; + /*0420*/ ISETP.LT.U32.AND P2, PT, R8, R7, PT ; + /*0430*/ @P2 SHF.R.U32.HI R5, RZ, 0x4, R5 ; + /*0440*/ @P2 IMAD.IADD R7, R7, 0x1, -R8 ; + /*0450*/ SHF.R.U32.HI R9, RZ, 0x1, R5 ; + /*0460*/ SGXT.U32 R2, R5, 0x1 ; + /*0470*/ SGXT.U32 R9, R9, 0x1 ; + /*0480*/ IMAD.IADD R2, R2, 0x1, R9 ; + /*0490*/ ISETP.LT.U32.AND P1, PT, R2, R7, PT ; + /*04a0*/ @P1 SHF.R.U32.HI R5, RZ, 0x2, R5 ; + /*04b0*/ @P1 IMAD.IADD R7, R7, 0x1, -R2 ; + /*04c0*/ SGXT.U32 R2, R5, 0x1 ; + /*04d0*/ SEL R5, RZ, 0x10, !P4 ; + /*04e0*/ ISETP.LT.U32.AND P4, PT, R2, R7, PT ; + /*04f0*/ NOP ; + /*0500*/ @P3 IADD3 R5, R5, 0x8, RZ ; + /*0510*/ @P2 IADD3 R5, R5, 0x4, RZ ; + /*0520*/ @P1 IADD3 R5, R5, 0x2, RZ ; + /*0530*/ ISETP.LT.AND P1, PT, R3, R4, PT ; + /*0540*/ NOP ; + /*0550*/ @P4 IADD3 R5, R5, 0x1, RZ ; + /*0560*/ @P0 IADD3 R5, -R5, 0x1f, RZ ; + /*0570*/ SEL R5, R5, 0xffffffff, !P1 ; + /*0580*/ STS [R10+0xc], R5 ; + /*0590*/ EXIT ; + /*05a0*/ BRA 0x5a0; + +// == dp4a_u32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IDP.4A.U8.U8 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == dp4a_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IDP.4A.S8.S8 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == dp4a_su +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IDP.4A.S8.U8 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == dp2a_lo +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IDP.2A.LO.S16.S8 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == dp2a_hi +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 7.152557373046875e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01f0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ LDS R6, [R8+0x8] ; + /*0200*/ NOP ; + /*0210*/ NOP ; + /*0220*/ IDP.2A.HI.U16.U8 R7, R0, R7, R6 ; + /*0230*/ STS [R8+0xc], R7 ; + /*0240*/ EXIT ; + /*0250*/ BRA 0x250; + +// == shfl_idx +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHFL.IDX PT, R7, R0, R7, 0x1f ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == shfl_up +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R9, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LOP3.LUT R7, R5, 0x1f, RZ, 0xc0, !PT ; + /*0200*/ NOP ; + /*0210*/ SHFL.UP P0, R7, R0, R7, RZ ; + /*0220*/ SEL R9, RZ, 0x1, !P0 ; + /*0230*/ STS [R10+0x8], R7 ; + /*0240*/ STS [R10+0xc], R9 ; + /*0250*/ EXIT ; + /*0260*/ BRA 0x260; + +// == shfl_down +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R9, c[0x0][0x160] +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R10, R10, 0x10, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LOP3.LUT R7, R5, 0x1f, RZ, 0xc0, !PT ; + /*0200*/ NOP ; + /*0210*/ SHFL.DOWN P0, R7, R0, R7, 0x1f ; + /*0220*/ SEL R9, RZ, 0x1, !P0 ; + /*0230*/ STS [R10+0x8], R7 ; + /*0240*/ STS [R10+0xc], R9 ; + /*0250*/ EXIT ; + /*0260*/ BRA 0x260; + +// == shfl_bfly +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHFL.BFLY PT, R7, R0, R7, 0x1f ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == shfl_bfly_imm +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ SHFL.BFLY PT, R7, R2, 0x1, 0x1f ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == shfl_idx_seg +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01d0 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x01e0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.U32 R8, R8, 0xc, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R0, [R8] ; + /*01e0*/ LDS R7, [R8+0x4] ; + /*01f0*/ NOP ; + /*0200*/ NOP ; + /*0210*/ SHFL.IDX PT, R7, R0, R7, 0x1807 ; + /*0220*/ STS [R8+0x8], R7 ; + /*0230*/ EXIT ; + /*0240*/ BRA 0x240; + +// == vote +// base register R14 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x4 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): IMAD R4, R3, 0x20, R4 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.MOV.U32 R5, RZ, RZ, 0x10 +// nop 0x01d0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R14, SR_TID.X ; + /*0010*/ IMAD.U32 R14, R14, 0x14, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R14] ; + /*01e0*/ NOP ; + /*01f0*/ NOP ; + /*0200*/ LOP3.LUT P0, RZ, R2, 0x1, RZ, 0xc0, !PT ; + /*0210*/ VOTE.ANY P1, P0 ; + /*0220*/ VOTE.ALL P2, P0 ; + /*0230*/ VOTE.EQ P3, P0 ; + /*0240*/ VOTE.ANY R7, PT, P0 ; + /*0250*/ SEL R9, RZ, 0x1, !P1 ; + /*0260*/ SEL R11, RZ, 0x1, !P2 ; + /*0270*/ STS [R14+0x4], R7 ; + /*0280*/ SEL R13, RZ, 0x1, !P3 ; + /*0290*/ STS [R14+0x8], R9 ; + /*02a0*/ STS [R14+0xc], R11 ; + /*02b0*/ STS [R14+0x10], R13 ; + /*02c0*/ EXIT ; + /*02d0*/ BRA 0x2d0; + +// == activemask +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0190 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x01a0 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01b0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ S2R R3, SR_CTAID.X ; + /*01a0*/ VOTE.ANY R7, PT, PT ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ NOP ; + /*01e0*/ NOP ; + /*01f0*/ LDS R2, [R8] ; + /*0200*/ STS [R8+0x4], R7 ; + /*0210*/ EXIT ; + /*0220*/ BRA 0x220; + +// == match_any +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ LOP3.LUT R0, R2, 0x3, RZ, 0xc0, !PT ; + /*0200*/ MATCH.ANY R7, R0 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == redux_add +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ REDUX.SUM UR6, R2 ; + /*0200*/ MOV R7, UR6 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == redux_min_s32 +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ REDUX.MIN.S32 UR6, R2 ; + /*0200*/ MOV R7, UR6 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == redux_or +// base register R8 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): HFMA2.MMA R5, -RZ, RZ, 0, 2.384185791015625e-07 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x01c0 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R8, SR_TID.X ; + /*0010*/ IMAD.SHL.U32 R8, R8, 0x8, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R2, [R8] ; + /*01e0*/ NOP ; + /*01f0*/ REDUX.OR UR6, R2 ; + /*0200*/ MOV R7, UR6 ; + /*0210*/ STS [R8+0x4], R7 ; + /*0220*/ EXIT ; + /*0230*/ BRA 0x230; + +// == hmma_16816_f16 +// base register R14 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x20 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0230 (feeds only the removed addressing): MOV R5, 0x8 +// nop 0x0240 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R14, SR_TID.X ; + /*0010*/ IMAD.U32 R14, R14, 0x28, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R8, [R14] ; + /*01e0*/ LDS R9, [R14+0x4] ; + /*01f0*/ LDS R10, [R14+0x8] ; + /*0200*/ LDS R11, [R14+0xc] ; + /*0210*/ LDS R6, [R14+0x10] ; + /*0220*/ LDS R7, [R14+0x14] ; + /*0230*/ LDS R12, [R14+0x18] ; + /*0240*/ LDS R13, [R14+0x1c] ; + /*0250*/ NOP ; + /*0260*/ NOP ; + /*0270*/ HMMA.16816.F16 R6, R8, R6, R12 ; + /*0280*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*0290*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02a0*/ STS [R14+0x20], R6 ; + /*02b0*/ STS [R14+0x24], R7 ; + /*02c0*/ EXIT ; + /*02d0*/ BRA 0x2d0; + +// == hmma_16816_f32 +// base register R16 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x28 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0250 (feeds only the removed addressing): MOV R5, 0x10 +// nop 0x0260 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R16, SR_TID.X ; + /*0010*/ IMAD.U32 R16, R16, 0x38, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R8, [R16] ; + /*01e0*/ LDS R9, [R16+0x4] ; + /*01f0*/ LDS R10, [R16+0x8] ; + /*0200*/ LDS R11, [R16+0xc] ; + /*0210*/ LDS R6, [R16+0x10] ; + /*0220*/ LDS R7, [R16+0x14] ; + /*0230*/ LDS R12, [R16+0x18] ; + /*0240*/ LDS R13, [R16+0x1c] ; + /*0250*/ LDS R14, [R16+0x20] ; + /*0260*/ LDS R15, [R16+0x24] ; + /*0270*/ NOP ; + /*0280*/ NOP ; + /*0290*/ HMMA.16816.F32 R8, R8, R6, R12 ; + /*02a0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02b0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02c0*/ STS [R16+0x28], R8 ; + /*02d0*/ STS [R16+0x2c], R9 ; + /*02e0*/ STS [R16+0x30], R10 ; + /*02f0*/ STS [R16+0x34], R11 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == hmma_1688_f16 +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x14 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0200 (feeds only the removed addressing): MOV R5, 0x8 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x1c, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R6, [R10] ; + /*01e0*/ LDS R7, [R10+0x4] ; + /*01f0*/ LDS R0, [R10+0x8] ; + /*0200*/ LDS R8, [R10+0xc] ; + /*0210*/ LDS R9, [R10+0x10] ; + /*0220*/ NOP ; + /*0230*/ NOP ; + /*0240*/ HMMA.1688.F16 R6, R6, R0, R8 ; + /*0250*/ STS [R10+0x14], R6 ; + /*0260*/ STS [R10+0x18], R7 ; + /*0270*/ EXIT ; + /*0280*/ BRA 0x280; + +// == hmma_16816_bf16 +// base register R16 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x28 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0250 (feeds only the removed addressing): MOV R5, 0x10 +// nop 0x0260 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R16, SR_TID.X ; + /*0010*/ IMAD.U32 R16, R16, 0x38, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R8, [R16] ; + /*01e0*/ LDS R9, [R16+0x4] ; + /*01f0*/ LDS R10, [R16+0x8] ; + /*0200*/ LDS R11, [R16+0xc] ; + /*0210*/ LDS R6, [R16+0x10] ; + /*0220*/ LDS R7, [R16+0x14] ; + /*0230*/ LDS R12, [R16+0x18] ; + /*0240*/ LDS R13, [R16+0x1c] ; + /*0250*/ LDS R14, [R16+0x20] ; + /*0260*/ LDS R15, [R16+0x24] ; + /*0270*/ NOP ; + /*0280*/ NOP ; + /*0290*/ HMMA.16816.F32.BF16 R8, R8, R6, R12 ; + /*02a0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02b0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02c0*/ STS [R16+0x28], R8 ; + /*02d0*/ STS [R16+0x2c], R9 ; + /*02e0*/ STS [R16+0x30], R10 ; + /*02f0*/ STS [R16+0x34], R11 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == hmma_1688_tf32 +// base register R16 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x28 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0250 (feeds only the removed addressing): MOV R5, 0x10 +// nop 0x0260 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R16, SR_TID.X ; + /*0010*/ IMAD.U32 R16, R16, 0x38, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R8, [R16] ; + /*01e0*/ LDS R9, [R16+0x4] ; + /*01f0*/ LDS R10, [R16+0x8] ; + /*0200*/ LDS R11, [R16+0xc] ; + /*0210*/ LDS R6, [R16+0x10] ; + /*0220*/ LDS R7, [R16+0x14] ; + /*0230*/ LDS R12, [R16+0x18] ; + /*0240*/ LDS R13, [R16+0x1c] ; + /*0250*/ LDS R14, [R16+0x20] ; + /*0260*/ LDS R15, [R16+0x24] ; + /*0270*/ NOP ; + /*0280*/ NOP ; + /*0290*/ HMMA.1688.F32.TF32 R8, R8, R6, R12 ; + /*02a0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02b0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02c0*/ STS [R16+0x28], R8 ; + /*02d0*/ STS [R16+0x2c], R9 ; + /*02e0*/ STS [R16+0x30], R10 ; + /*02f0*/ STS [R16+0x34], R11 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == imma_16832_s8 +// base register R16 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0160 (feeds only the removed addressing): MOV R5, 0x28 +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R4, R3, R4, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R4, R5, c[0x0][0x160] +// nop 0x0250 (feeds only the removed addressing): MOV R5, 0x10 +// nop 0x0260 (feeds only the removed addressing): IMAD.WIDE.U32 R4, R4, R5, c[0x0][0x168] + /*0000*/ S2R R16, SR_TID.X ; + /*0010*/ IMAD.U32 R16, R16, 0x38, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R4, SR_TID.X ; + /*0180*/ NOP ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R8, [R16] ; + /*01e0*/ LDS R9, [R16+0x4] ; + /*01f0*/ LDS R10, [R16+0x8] ; + /*0200*/ LDS R11, [R16+0xc] ; + /*0210*/ LDS R6, [R16+0x10] ; + /*0220*/ LDS R7, [R16+0x14] ; + /*0230*/ LDS R12, [R16+0x18] ; + /*0240*/ LDS R13, [R16+0x1c] ; + /*0250*/ LDS R14, [R16+0x20] ; + /*0260*/ LDS R15, [R16+0x24] ; + /*0270*/ NOP ; + /*0280*/ NOP ; + /*0290*/ IMMA.16832.S8.S8 R8, R8.ROW, R6.COL, R12 ; + /*02a0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02b0*/ @!UPT UIADD3 URZ, URZ, URZ, URZ ; + /*02c0*/ STS [R16+0x28], R8 ; + /*02d0*/ STS [R16+0x2c], R9 ; + /*02e0*/ STS [R16+0x30], R10 ; + /*02f0*/ STS [R16+0x34], R11 ; + /*0300*/ EXIT ; + /*0310*/ BRA 0x310; + +// == dadd +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R0, R3, R0, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R0, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 9.5367431640625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ LDS R7, [R10+0xc] ; + /*0210*/ NOP ; + /*0220*/ DADD R4, R4, R6 ; + /*0230*/ NOP ; + /*0240*/ STS [R10+0x10], R4 ; + /*0250*/ STS [R10+0x14], R5 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; + +// == dmul +// base register R10 +// nop 0x0000..0x0140: the JIT's barrier set-up +// nop 0x0140 (feeds only the removed addressing): MOV R1, c[0x0][0x28] +// nop 0x0170 (feeds only the removed addressing): ULDC.64 UR4, c[0x0][0x118] +// nop 0x0190 (feeds only the removed addressing): LEA R0, R3, R0, 0x5 +// nop 0x01a0 (feeds only the removed addressing): IMAD.WIDE.U32 R2, R0, R5, c[0x0][0x160] +// nop 0x01f0 (feeds only the removed addressing): HFMA2.MMA R9, -RZ, RZ, 0, 4.76837158203125e-07 +// nop 0x0210 (feeds only the removed addressing): IMAD.WIDE.U32 R6, R0, R9, c[0x0][0x168] + /*0000*/ S2R R10, SR_TID.X ; + /*0010*/ IMAD.U32 R10, R10, 0x18, RZ ; + /*0020*/ NOP ; + /*0030*/ NOP ; + /*0040*/ NOP ; + /*0050*/ NOP ; + /*0060*/ NOP ; + /*0070*/ NOP ; + /*0080*/ NOP ; + /*0090*/ NOP ; + /*00a0*/ NOP ; + /*00b0*/ NOP ; + /*00c0*/ NOP ; + /*00d0*/ NOP ; + /*00e0*/ NOP ; + /*00f0*/ NOP ; + /*0100*/ NOP ; + /*0110*/ NOP ; + /*0120*/ NOP ; + /*0130*/ NOP ; + /*0140*/ NOP ; + /*0150*/ NOP ; + /*0160*/ NOP ; + /*0170*/ S2R R0, SR_TID.X ; + /*0180*/ HFMA2.MMA R5, -RZ, RZ, 0, 9.5367431640625e-07 ; + /*0190*/ NOP ; + /*01a0*/ S2R R3, SR_CTAID.X ; + /*01b0*/ NOP ; + /*01c0*/ NOP ; + /*01d0*/ LDS R4, [R10] ; + /*01e0*/ LDS R5, [R10+0x4] ; + /*01f0*/ LDS R6, [R10+0x8] ; + /*0200*/ LDS R7, [R10+0xc] ; + /*0210*/ NOP ; + /*0220*/ DMUL R4, R4, R6 ; + /*0230*/ NOP ; + /*0240*/ STS [R10+0x10], R4 ; + /*0250*/ STS [R10+0x14], R5 ; + /*0260*/ EXIT ; + /*0270*/ BRA 0x270; diff --git a/dm/gfx/tests/ampere_orin/ampere_orin.u32.gz b/dm/gfx/tests/ampere_orin/ampere_orin.u32.gz new file mode 100644 index 0000000000..3482d102ad Binary files /dev/null and b/dm/gfx/tests/ampere_orin/ampere_orin.u32.gz differ diff --git a/dm/gfx/tests/ampere_orin/manifest.json b/dm/gfx/tests/ampere_orin/manifest.json new file mode 100644 index 0000000000..3d238c704f --- /dev/null +++ b/dm/gfx/tests/ampere_orin/manifest.json @@ -0,0 +1,3099 @@ +{ + "device": { + "gpu": "Orin", + "sm": "87", + "cuda_driver": "12.6", + "nvdisasm": "Build cuda_12.6.r12.6/compiler.34841621_0" + }, + "arch": "sm_87", + "ops": [ + { + "name": "fadd_rn", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rn.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 0, + 800 + ], + "rows": [ + 800, + 12288 + ] + }, + { + "name": "fmul_rn", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rn.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 13088, + 800 + ], + "rows": [ + 13888, + 12288 + ] + }, + { + "name": "ffma_rn", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rn.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 26176, + 800 + ], + "rows": [ + 26976, + 16384 + ] + }, + { + "name": "fadd_rz", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rz.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 43360, + 800 + ], + "rows": [ + 44160, + 12288 + ] + }, + { + "name": "fmul_rz", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rz.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 56448, + 800 + ], + "rows": [ + 57248, + 12288 + ] + }, + { + "name": "ffma_rz", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rz.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 69536, + 800 + ], + "rows": [ + 70336, + 16384 + ] + }, + { + "name": "fadd_rm", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rm.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 86720, + 800 + ], + "rows": [ + 87520, + 12288 + ] + }, + { + "name": "fmul_rm", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rm.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 99808, + 800 + ], + "rows": [ + 100608, + 12288 + ] + }, + { + "name": "ffma_rm", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rm.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 112896, + 800 + ], + "rows": [ + 113696, + 16384 + ] + }, + { + "name": "fadd_rp", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rp.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 130080, + 800 + ], + "rows": [ + 130880, + 12288 + ] + }, + { + "name": "fmul_rp", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rp.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 143168, + 800 + ], + "rows": [ + 143968, + 12288 + ] + }, + { + "name": "ffma_rp", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rp.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 156256, + 800 + ], + "rows": [ + 157056, + 16384 + ] + }, + { + "name": "fadd_ftz", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rn.ftz.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 173440, + 800 + ], + "rows": [ + 174240, + 12288 + ] + }, + { + "name": "fadd_sat", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; add.rn.sat.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 186528, + 800 + ], + "rows": [ + 187328, + 12288 + ] + }, + { + "name": "fsub", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; sub.rn.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 199616, + 800 + ], + "rows": [ + 200416, + 12288 + ] + }, + { + "name": "fmul_ftz", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rn.ftz.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 212704, + 800 + ], + "rows": [ + 213504, + 12288 + ] + }, + { + "name": "fmul_sat", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mul.rn.sat.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 225792, + 800 + ], + "rows": [ + 226592, + 12288 + ] + }, + { + "name": "ffma_ftz", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rn.ftz.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 238880, + 800 + ], + "rows": [ + 239680, + 16384 + ] + }, + { + "name": "ffma_sat", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rn.sat.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 256064, + 800 + ], + "rows": [ + 256864, + 16384 + ] + }, + { + "name": "ffma_ftz_sat", + "family": "f32 arith", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; fma.rn.ftz.sat.f32 %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;", + "prog": [ + 273248, + 800 + ], + "rows": [ + 274048, + 16384 + ] + }, + { + "name": "fadd_abs_neg", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; abs.f32 %f2, %f0; neg.f32 %f3, %f1; add.rn.f32 %f4, %f2, %f3; mov.b32 %o0, %f4;", + "prog": [ + 290432, + 800 + ], + "rows": [ + 291232, + 12288 + ] + }, + { + "name": "fmul_by_imm", + "family": "f32 arith", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mul.rn.f32 %f2, %f0, 0f3EAAAAAB; mov.b32 %o0, %f2;", + "prog": [ + 303520, + 800 + ], + "rows": [ + 304320, + 8192 + ] + }, + { + "name": "fmin", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; min.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 312512, + 800 + ], + "rows": [ + 313312, + 12288 + ] + }, + { + "name": "fmax", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; max.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 325600, + 800 + ], + "rows": [ + 326400, + 12288 + ] + }, + { + "name": "fmin_ftz", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; min.ftz.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 338688, + 800 + ], + "rows": [ + 339488, + 12288 + ] + }, + { + "name": "fmax_nan", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; max.NaN.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 351776, + 800 + ], + "rows": [ + 352576, + 12288 + ] + }, + { + "name": "fmin_nan", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; min.NaN.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 364864, + 800 + ], + "rows": [ + 365664, + 12288 + ] + }, + { + "name": "fcopysign", + "family": "f32 arith", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; copysign.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 377952, + 800 + ], + "rows": [ + 378752, + 12288 + ] + }, + { + "name": "fabs", + "family": "f32 arith", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; abs.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 391040, + 800 + ], + "rows": [ + 391840, + 8192 + ] + }, + { + "name": "fneg", + "family": "f32 arith", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; neg.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 400032, + 800 + ], + "rows": [ + 400832, + 8192 + ] + }, + { + "name": "fsetp", + "family": "f32 compare", + "ins": "ff", + "outs": "uuuuuuuuuuuuuu", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; setp.eq.f32 %p0, %f0, %f1; selp.u32 %o0, 1, 0, %p0; setp.ne.f32 %p0, %f0, %f1; selp.u32 %o1, 1, 0, %p0; setp.lt.f32 %p0, %f0, %f1; selp.u32 %o2, 1, 0, %p0; setp.le.f32 %p0, %f0, %f1; selp.u32 %o3, 1, 0, %p0; setp.gt.f32 %p0, %f0, %f1; selp.u32 %o4, 1, 0, %p0; setp.ge.f32 %p0, %f0, %f1; selp.u32 %o5, 1, 0, %p0; setp.equ.f32 %p0, %f0, %f1; selp.u32 %o6, 1, 0, %p0; setp.neu.f32 %p0, %f0, %f1; selp.u32 %o7, 1, 0, %p0; setp.ltu.f32 %p0, %f0, %f1; selp.u32 %o8, 1, 0, %p0; setp.leu.f32 %p0, %f0, %f1; selp.u32 %o9, 1, 0, %p0; setp.gtu.f32 %p0, %f0, %f1; selp.u32 %o10, 1, 0, %p0; setp.geu.f32 %p0, %f0, %f1; selp.u32 %o11, 1, 0, %p0; setp.num.f32 %p0, %f0, %f1; selp.u32 %o12, 1, 0, %p0; setp.nan.f32 %p0, %f0, %f1; selp.u32 %o13, 1, 0, %p0;", + "prog": [ + 409024, + 1312 + ], + "rows": [ + 410336, + 65536 + ] + }, + { + "name": "fsetp_ftz", + "family": "f32 compare", + "ins": "ff", + "outs": "uuuu", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; setp.eq.ftz.f32 %p0, %f0, %f1; selp.u32 %o0, 1, 0, %p0; setp.lt.ftz.f32 %p0, %f0, %f1; selp.u32 %o1, 1, 0, %p0; setp.gtu.ftz.f32 %p0, %f0, %f1; selp.u32 %o2, 1, 0, %p0; setp.ne.ftz.f32 %p0, %f0, %f1; selp.u32 %o3, 1, 0, %p0;", + "prog": [ + 475872, + 928 + ], + "rows": [ + 476800, + 24576 + ] + }, + { + "name": "fset_bf", + "family": "f32 compare", + "ins": "ff", + "outs": "ff", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; set.lt.f32.f32 %f2, %f0, %f1; set.neu.f32.f32 %f3, %f0, %f1; mov.b32 %o0, %f2; mov.b32 %o1, %f3;", + "prog": [ + 501376, + 800 + ], + "rows": [ + 502176, + 16384 + ] + }, + { + "name": "fset_u32", + "family": "f32 compare", + "ins": "ff", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; set.ge.u32.f32 %o0, %f0, %f1; set.nan.u32.f32 %o1, %f0, %f1;", + "prog": [ + 518560, + 800 + ], + "rows": [ + 519360, + 16384 + ] + }, + { + "name": "ftestp", + "family": "f32 compare", + "ins": "f", + "outs": "uuuuu", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; testp.finite.f32 %p0, %f0; selp.u32 %o0, 1, 0, %p0; testp.infinite.f32 %p0, %f0; selp.u32 %o1, 1, 0, %p0; testp.number.f32 %p0, %f0; selp.u32 %o2, 1, 0, %p0; testp.normal.f32 %p0, %f0; selp.u32 %o3, 1, 0, %p0; testp.subnormal.f32 %p0, %f0; selp.u32 %o4, 1, 0, %p0;", + "prog": [ + 535744, + 1056 + ], + "rows": [ + 536800, + 24576 + ] + }, + { + "name": "fsel_cmp", + "family": "f32 compare", + "ins": "fff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; setp.gt.f32 %p0, %f0, 0f00000000; selp.f32 %f2, %f1, %f3, %p0; mov.b32 %o0, %f2;", + "prog": [ + 561376, + 800 + ], + "rows": [ + 562176, + 16384 + ] + }, + { + "name": "mufu_ex2", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; ex2.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 578560, + 800 + ], + "rows": [ + 579360, + 8192 + ] + }, + { + "name": "mufu_ex2_ftz", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; ex2.approx.ftz.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 587552, + 800 + ], + "rows": [ + 588352, + 8192 + ] + }, + { + "name": "mufu_lg2", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; lg2.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 596544, + 800 + ], + "rows": [ + 597344, + 8192 + ] + }, + { + "name": "mufu_lg2_ftz", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; lg2.approx.ftz.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 605536, + 800 + ], + "rows": [ + 606336, + 8192 + ] + }, + { + "name": "mufu_rcp", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; rcp.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 614528, + 800 + ], + "rows": [ + 615328, + 8192 + ] + }, + { + "name": "mufu_rcp_ftz", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; rcp.approx.ftz.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 623520, + 800 + ], + "rows": [ + 624320, + 8192 + ] + }, + { + "name": "mufu_rsqrt", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; rsqrt.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 632512, + 800 + ], + "rows": [ + 633312, + 8192 + ] + }, + { + "name": "mufu_rsqrt_ftz", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; rsqrt.approx.ftz.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 641504, + 800 + ], + "rows": [ + 642304, + 8192 + ] + }, + { + "name": "mufu_sqrt", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; sqrt.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 650496, + 800 + ], + "rows": [ + 651296, + 8192 + ] + }, + { + "name": "mufu_sqrt_ftz", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; sqrt.approx.ftz.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 659488, + 800 + ], + "rows": [ + 660288, + 8192 + ] + }, + { + "name": "mufu_sin", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; sin.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 668480, + 800 + ], + "rows": [ + 669280, + 8192 + ] + }, + { + "name": "mufu_cos", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cos.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 677472, + 800 + ], + "rows": [ + 678272, + 8192 + ] + }, + { + "name": "mufu_tanh", + "family": "mufu", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; tanh.approx.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 686464, + 800 + ], + "rows": [ + 687264, + 8192 + ] + }, + { + "name": "mufu_ex2_f16x2", + "family": "mufu", + "ins": "h", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; ex2.approx.f16x2 %x2, %x0; mov.b32 %o0, %x2;", + "prog": [ + 695456, + 800 + ], + "rows": [ + 696256, + 8192 + ] + }, + { + "name": "mufu_tanh_f16x2", + "family": "mufu", + "ins": "h", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; tanh.approx.f16x2 %x2, %x0; mov.b32 %o0, %x2;", + "prog": [ + 704448, + 800 + ], + "rows": [ + 705248, + 8192 + ] + }, + { + "name": "fdiv_rn", + "family": "f32 precise", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; div.rn.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 713440, + 928 + ], + "rows": [ + 714368, + 12288 + ] + }, + { + "name": "fdiv_full", + "family": "f32 precise", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; div.full.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 726656, + 928 + ], + "rows": [ + 727584, + 12288 + ] + }, + { + "name": "fdiv_approx", + "family": "f32 precise", + "ins": "ff", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; div.approx.f32 %f2, %f0, %f1; mov.b32 %o0, %f2;", + "prog": [ + 739872, + 800 + ], + "rows": [ + 740672, + 12288 + ] + }, + { + "name": "frcp_rn", + "family": "f32 precise", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; rcp.rn.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 752960, + 928 + ], + "rows": [ + 753888, + 8192 + ] + }, + { + "name": "fsqrt_rn", + "family": "f32 precise", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; sqrt.rn.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 762080, + 928 + ], + "rows": [ + 763008, + 8192 + ] + }, + { + "name": "f2i_s32_rni", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rni.s32.f32 %o0, %f0;", + "prog": [ + 771200, + 800 + ], + "rows": [ + 772000, + 8192 + ] + }, + { + "name": "frnd_rni", + "family": "convert", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rni.f32.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 780192, + 800 + ], + "rows": [ + 780992, + 8192 + ] + }, + { + "name": "f2i_s32_rzi", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.s32.f32 %o0, %f0;", + "prog": [ + 789184, + 800 + ], + "rows": [ + 789984, + 8192 + ] + }, + { + "name": "frnd_rzi", + "family": "convert", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.f32.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 798176, + 800 + ], + "rows": [ + 798976, + 8192 + ] + }, + { + "name": "f2i_s32_rmi", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rmi.s32.f32 %o0, %f0;", + "prog": [ + 807168, + 800 + ], + "rows": [ + 807968, + 8192 + ] + }, + { + "name": "frnd_rmi", + "family": "convert", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rmi.f32.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 816160, + 800 + ], + "rows": [ + 816960, + 8192 + ] + }, + { + "name": "f2i_s32_rpi", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rpi.s32.f32 %o0, %f0;", + "prog": [ + 825152, + 800 + ], + "rows": [ + 825952, + 8192 + ] + }, + { + "name": "frnd_rpi", + "family": "convert", + "ins": "f", + "outs": "f", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rpi.f32.f32 %f2, %f0; mov.b32 %o0, %f2;", + "prog": [ + 834144, + 800 + ], + "rows": [ + 834944, + 8192 + ] + }, + { + "name": "f2i_u32_rzi", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.u32.f32 %o0, %f0;", + "prog": [ + 843136, + 800 + ], + "rows": [ + 843936, + 8192 + ] + }, + { + "name": "f2i_s32_ftz", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.ftz.s32.f32 %o0, %f0;", + "prog": [ + 852128, + 800 + ], + "rows": [ + 852928, + 8192 + ] + }, + { + "name": "f2i_s16_sat", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.sat.s16.f32 %w0, %f0; cvt.s32.s16 %o0, %w0;", + "prog": [ + 861120, + 800 + ], + "rows": [ + 861920, + 8192 + ] + }, + { + "name": "f2i_u8_sat", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rni.sat.u8.f32 %w0, %f0; cvt.u32.u8 %o0, %w0;", + "prog": [ + 870112, + 800 + ], + "rows": [ + 870912, + 8192 + ] + }, + { + "name": "f2i_s64", + "family": "convert", + "ins": "f", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rzi.s64.f32 %d0, %f0; mov.b64 {%o0, %o1}, %d0;", + "prog": [ + 879104, + 800 + ], + "rows": [ + 879904, + 12288 + ] + }, + { + "name": "i2f_s32_rn", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.rn.f32.s32 %f2, %i0; mov.b32 %o0, %f2;", + "prog": [ + 892192, + 800 + ], + "rows": [ + 892992, + 8192 + ] + }, + { + "name": "i2f_s32_rz", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.rz.f32.s32 %f2, %i0; mov.b32 %o0, %f2;", + "prog": [ + 901184, + 800 + ], + "rows": [ + 901984, + 8192 + ] + }, + { + "name": "i2f_s32_rm", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.rm.f32.s32 %f2, %i0; mov.b32 %o0, %f2;", + "prog": [ + 910176, + 800 + ], + "rows": [ + 910976, + 8192 + ] + }, + { + "name": "i2f_s32_rp", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.rp.f32.s32 %f2, %i0; mov.b32 %o0, %f2;", + "prog": [ + 919168, + 800 + ], + "rows": [ + 919968, + 8192 + ] + }, + { + "name": "i2f_u32_rn", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.rn.f32.u32 %f2, %i0; mov.b32 %o0, %f2;", + "prog": [ + 928160, + 800 + ], + "rows": [ + 928960, + 8192 + ] + }, + { + "name": "i2f_s16", + "family": "convert", + "ins": "u", + "outs": "f", + "cases": 1024, + "ptx": "cvt.u16.u32 %w0, %i0; cvt.rn.f32.s16 %f2, %w0; mov.b32 %o0, %f2;", + "prog": [ + 937152, + 800 + ], + "rows": [ + 937952, + 8192 + ] + }, + { + "name": "i2f_s64", + "family": "convert", + "ins": "uu", + "outs": "f", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; cvt.rn.f32.s64 %f2, %d0; mov.b32 %o0, %f2;", + "prog": [ + 946144, + 800 + ], + "rows": [ + 946944, + 12288 + ] + }, + { + "name": "f2f_f16_rn", + "family": "convert", + "ins": "f", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rn.f16.f32 %y0, %f0; mov.b16 %w0, %y0; cvt.u32.u16 %o0, %w0;", + "prog": [ + 959232, + 800 + ], + "rows": [ + 960032, + 8192 + ] + }, + { + "name": "f2f_f16_rz", + "family": "convert", + "ins": "f", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rz.f16.f32 %y0, %f0; mov.b16 %w0, %y0; cvt.u32.u16 %o0, %w0;", + "prog": [ + 968224, + 800 + ], + "rows": [ + 969024, + 8192 + ] + }, + { + "name": "f2f_f32_from_f16", + "family": "convert", + "ins": "h", + "outs": "ff", + "cases": 1024, + "ptx": "mov.b32 {%y0, %y1}, %i0; cvt.f32.f16 %f2, %y0; cvt.f32.f16 %f3, %y1; mov.b32 %o0, %f2; mov.b32 %o1, %f3;", + "prog": [ + 977216, + 800 + ], + "rows": [ + 978016, + 12288 + ] + }, + { + "name": "f2fp_pack_f16x2", + "family": "convert", + "ins": "ff", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.f16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;", + "prog": [ + 990304, + 800 + ], + "rows": [ + 991104, + 12288 + ] + }, + { + "name": "f2fp_pack_f16x2_relu", + "family": "convert", + "ins": "ff", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.relu.f16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;", + "prog": [ + 1003392, + 800 + ], + "rows": [ + 1004192, + 12288 + ] + }, + { + "name": "f2fp_pack_bf16x2", + "family": "convert", + "ins": "ff", + "outs": "b", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.bf16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;", + "prog": [ + 1016480, + 800 + ], + "rows": [ + 1017280, + 12288 + ] + }, + { + "name": "f2f_tf32", + "family": "convert", + "ins": "f", + "outs": "u", + "cases": 1024, + "ptx": "mov.b32 %f0, %i0; cvt.rna.tf32.f32 %o0, %f0;", + "prog": [ + 1029568, + 800 + ], + "rows": [ + 1030368, + 8192 + ] + }, + { + "name": "i2i_s8_sat", + "family": "convert", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "cvt.sat.s8.s32 %w0, %i0; cvt.s32.s8 %o0, %w0;", + "prog": [ + 1038560, + 800 + ], + "rows": [ + 1039360, + 8192 + ] + }, + { + "name": "i2i_u16_sat", + "family": "convert", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "cvt.sat.u16.s32 %w0, %i0; cvt.u32.u16 %o0, %w0;", + "prog": [ + 1047552, + 800 + ], + "rows": [ + 1048352, + 8192 + ] + }, + { + "name": "sext_s8", + "family": "convert", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "cvt.s32.s8 %o0, %i0;", + "prog": [ + 1056544, + 800 + ], + "rows": [ + 1057344, + 8192 + ] + }, + { + "name": "hadd2", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; add.rn.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1065536, + 800 + ], + "rows": [ + 1066336, + 12288 + ] + }, + { + "name": "hadd2_ftz", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; add.rn.ftz.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1078624, + 800 + ], + "rows": [ + 1079424, + 12288 + ] + }, + { + "name": "hadd2_sat", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; add.rn.sat.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1091712, + 800 + ], + "rows": [ + 1092512, + 12288 + ] + }, + { + "name": "hsub2", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; sub.rn.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1104800, + 800 + ], + "rows": [ + 1105600, + 12288 + ] + }, + { + "name": "hmul2", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mul.rn.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1117888, + 800 + ], + "rows": [ + 1118688, + 12288 + ] + }, + { + "name": "hmul2_ftz", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mul.rn.ftz.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1130976, + 800 + ], + "rows": [ + 1131776, + 12288 + ] + }, + { + "name": "hfma2", + "family": "f16x2", + "ins": "hhh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; fma.rn.f16x2 %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;", + "prog": [ + 1144064, + 800 + ], + "rows": [ + 1144864, + 16384 + ] + }, + { + "name": "hfma2_ftz", + "family": "f16x2", + "ins": "hhh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; fma.rn.ftz.f16x2 %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;", + "prog": [ + 1161248, + 800 + ], + "rows": [ + 1162048, + 16384 + ] + }, + { + "name": "hfma2_sat", + "family": "f16x2", + "ins": "hhh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; fma.rn.sat.f16x2 %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;", + "prog": [ + 1178432, + 800 + ], + "rows": [ + 1179232, + 16384 + ] + }, + { + "name": "hfma2_relu", + "family": "f16x2", + "ins": "hhh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; fma.rn.relu.f16x2 %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;", + "prog": [ + 1195616, + 800 + ], + "rows": [ + 1196416, + 16384 + ] + }, + { + "name": "hmin2", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; min.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1212800, + 800 + ], + "rows": [ + 1213600, + 12288 + ] + }, + { + "name": "hmax2", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; max.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1225888, + 800 + ], + "rows": [ + 1226688, + 12288 + ] + }, + { + "name": "hmin2_nan", + "family": "f16x2", + "ins": "hh", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; min.NaN.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1238976, + 800 + ], + "rows": [ + 1239776, + 12288 + ] + }, + { + "name": "hneg2", + "family": "f16x2", + "ins": "h", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; neg.f16x2 %x2, %x0; mov.b32 %o0, %x2;", + "prog": [ + 1252064, + 800 + ], + "rows": [ + 1252864, + 8192 + ] + }, + { + "name": "habs2", + "family": "f16x2", + "ins": "h", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; abs.f16x2 %x2, %x0; mov.b32 %o0, %x2;", + "prog": [ + 1261056, + 800 + ], + "rows": [ + 1261856, + 8192 + ] + }, + { + "name": "hadd2_imm", + "family": "f16x2", + "ins": "h", + "outs": "h", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, 0x3C00BC00; add.rn.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1270048, + 800 + ], + "rows": [ + 1270848, + 8192 + ] + }, + { + "name": "hadd_f16", + "family": "f16x2", + "ins": "hh", + "outs": "hh", + "cases": 1024, + "ptx": "mov.b32 {%y0, %y1}, %i0; mov.b32 {%y2, %y3}, %i1; add.rn.f16 %y4, %y0, %y2; mul.rn.f16 %y5, %y1, %y3; mov.b16 %w0, %y4; cvt.u32.u16 %o0, %w0; mov.b16 %w1, %y5; cvt.u32.u16 %o1, %w1;", + "prog": [ + 1279040, + 800 + ], + "rows": [ + 1279840, + 16384 + ] + }, + { + "name": "hsetp2", + "family": "f16x2", + "ins": "hh", + "outs": "uuuuuuuu", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; setp.lt.f16x2 %p0|%p1, %x0, %x1; selp.u32 %o0, 1, 0, %p0; selp.u32 %o1, 1, 0, %p1; setp.eq.f16x2 %p0|%p1, %x0, %x1; selp.u32 %o2, 1, 0, %p0; selp.u32 %o3, 1, 0, %p1; setp.gtu.f16x2 %p0|%p1, %x0, %x1; selp.u32 %o4, 1, 0, %p0; selp.u32 %o5, 1, 0, %p1; setp.nan.f16x2 %p0|%p1, %x0, %x1; selp.u32 %o6, 1, 0, %p0; selp.u32 %o7, 1, 0, %p1;", + "prog": [ + 1296224, + 1056 + ], + "rows": [ + 1297280, + 40960 + ] + }, + { + "name": "hset2", + "family": "f16x2", + "ins": "hh", + "outs": "hu", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; set.lt.f16x2.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2; set.geu.u32.f16x2 %o1, %x0, %x1;", + "prog": [ + 1338240, + 800 + ], + "rows": [ + 1339040, + 16384 + ] + }, + { + "name": "bfma2", + "family": "bf16x2", + "ins": "bbb", + "outs": "b", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; fma.rn.bf16x2 %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;", + "prog": [ + 1355424, + 800 + ], + "rows": [ + 1356224, + 16384 + ] + }, + { + "name": "bmin2", + "family": "bf16x2", + "ins": "bb", + "outs": "b", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; min.bf16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1372608, + 800 + ], + "rows": [ + 1373408, + 12288 + ] + }, + { + "name": "bmax2_nan", + "family": "bf16x2", + "ins": "bb", + "outs": "b", + "cases": 1024, + "ptx": "mov.b32 %x0, %i0; mov.b32 %x1, %i1; max.NaN.bf16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;", + "prog": [ + 1385696, + 800 + ], + "rows": [ + 1386496, + 12288 + ] + }, + { + "name": "iadd", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "add.u32 %o0, %i0, %i1;", + "prog": [ + 1398784, + 800 + ], + "rows": [ + 1399584, + 12288 + ] + }, + { + "name": "iadd3", + "family": "int", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "add.u32 %t0, %i0, %i1; sub.u32 %o0, %t0, %i2;", + "prog": [ + 1411872, + 800 + ], + "rows": [ + 1412672, + 16384 + ] + }, + { + "name": "iadd_sat", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "add.sat.s32 %o0, %i0, %i1;", + "prog": [ + 1429056, + 800 + ], + "rows": [ + 1429856, + 12288 + ] + }, + { + "name": "isub_sat", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "sub.sat.s32 %o0, %i0, %i1;", + "prog": [ + 1442144, + 800 + ], + "rows": [ + 1442944, + 12288 + ] + }, + { + "name": "iadd64", + "family": "int", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; add.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;", + "prog": [ + 1455232, + 800 + ], + "rows": [ + 1456032, + 24576 + ] + }, + { + "name": "isub64", + "family": "int", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; sub.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;", + "prog": [ + 1480608, + 800 + ], + "rows": [ + 1481408, + 24576 + ] + }, + { + "name": "imul_lo", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mul.lo.u32 %o0, %i0, %i1;", + "prog": [ + 1505984, + 800 + ], + "rows": [ + 1506784, + 12288 + ] + }, + { + "name": "imul_hi_u32", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mul.hi.u32 %o0, %i0, %i1;", + "prog": [ + 1519072, + 800 + ], + "rows": [ + 1519872, + 12288 + ] + }, + { + "name": "imul_hi_s32", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mul.hi.s32 %o0, %i0, %i1;", + "prog": [ + 1532160, + 800 + ], + "rows": [ + 1532960, + 12288 + ] + }, + { + "name": "imul_wide_s32", + "family": "int", + "ins": "uu", + "outs": "uu", + "cases": 1024, + "ptx": "mul.wide.s32 %d0, %i0, %i1; mov.b64 {%o0, %o1}, %d0;", + "prog": [ + 1545248, + 800 + ], + "rows": [ + 1546048, + 16384 + ] + }, + { + "name": "imul_wide_u32", + "family": "int", + "ins": "uu", + "outs": "uu", + "cases": 1024, + "ptx": "mul.wide.u32 %d0, %i0, %i1; mov.b64 {%o0, %o1}, %d0;", + "prog": [ + 1562432, + 800 + ], + "rows": [ + 1563232, + 16384 + ] + }, + { + "name": "imad_lo", + "family": "int", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "mad.lo.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 1579616, + 800 + ], + "rows": [ + 1580416, + 16384 + ] + }, + { + "name": "imad_hi_s32", + "family": "int", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "mad.hi.s32 %o0, %i0, %i1, %i2;", + "prog": [ + 1596800, + 800 + ], + "rows": [ + 1597600, + 16384 + ] + }, + { + "name": "imul64", + "family": "int", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; mul.lo.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;", + "prog": [ + 1613984, + 928 + ], + "rows": [ + 1614912, + 24576 + ] + }, + { + "name": "imul24", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mul24.lo.s32 %o0, %i0, %i1;", + "prog": [ + 1639488, + 800 + ], + "rows": [ + 1640288, + 12288 + ] + }, + { + "name": "imin_s32", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "min.s32 %o0, %i0, %i1;", + "prog": [ + 1652576, + 800 + ], + "rows": [ + 1653376, + 12288 + ] + }, + { + "name": "imax_u32", + "family": "int", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "max.u32 %o0, %i0, %i1;", + "prog": [ + 1665664, + 800 + ], + "rows": [ + 1666464, + 12288 + ] + }, + { + "name": "imin_s64", + "family": "int", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; min.s64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;", + "prog": [ + 1678752, + 928 + ], + "rows": [ + 1679680, + 24576 + ] + }, + { + "name": "iabs", + "family": "int", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "abs.s32 %o0, %i0;", + "prog": [ + 1704256, + 800 + ], + "rows": [ + 1705056, + 8192 + ] + }, + { + "name": "ineg", + "family": "int", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "neg.s32 %o0, %i0;", + "prog": [ + 1713248, + 800 + ], + "rows": [ + 1714048, + 8192 + ] + }, + { + "name": "isad", + "family": "int", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "sad.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 1722240, + 800 + ], + "rows": [ + 1723040, + 16384 + ] + }, + { + "name": "idiv_u32", + "family": "int div", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "div.u32 %o0, %i0, %i1;", + "prog": [ + 1739424, + 1056 + ], + "rows": [ + 1740480, + 12288 + ] + }, + { + "name": "irem_u32", + "family": "int div", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "rem.u32 %o0, %i0, %i1;", + "prog": [ + 1752768, + 1056 + ], + "rows": [ + 1753824, + 12288 + ] + }, + { + "name": "idiv_s32", + "family": "int div", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "div.s32 %o0, %i0, %i1;", + "prog": [ + 1766112, + 1184 + ], + "rows": [ + 1767296, + 12288 + ] + }, + { + "name": "irem_s32", + "family": "int div", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "rem.s32 %o0, %i0, %i1;", + "prog": [ + 1779584, + 1184 + ], + "rows": [ + 1780768, + 12288 + ] + }, + { + "name": "idiv_u32_by_7", + "family": "int div", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "div.u32 %o0, %i0, 7;", + "prog": [ + 1793056, + 928 + ], + "rows": [ + 1793984, + 8192 + ] + }, + { + "name": "isetp", + "family": "int compare", + "ins": "uu", + "outs": "uuuuuu", + "cases": 1024, + "ptx": "setp.lt.s32 %p0, %i0, %i1; selp.u32 %o0, 1, 0, %p0; setp.lt.u32 %p0, %i0, %i1; selp.u32 %o1, 1, 0, %p0; setp.ge.s32 %p0, %i0, %i1; selp.u32 %o2, 1, 0, %p0; setp.hi.u32 %p0, %i0, %i1; selp.u32 %o3, 1, 0, %p0; setp.eq.s32 %p0, %i0, %i1; selp.u32 %o4, 1, 0, %p0; setp.ne.u32 %p0, %i0, %i1; selp.u32 %o5, 1, 0, %p0;", + "prog": [ + 1802176, + 1056 + ], + "rows": [ + 1803232, + 32768 + ] + }, + { + "name": "isetp_and", + "family": "int compare", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "setp.lt.s32 %p1, %i2, %i3; setp.lt.and.s32 %p0, %i0, %i1, %p1; selp.u32 %o0, 1, 0, %p0; setp.ge.or.u32 %p2, %i0, %i1, %p1; selp.u32 %o1, 1, 0, %p2;", + "prog": [ + 1836000, + 928 + ], + "rows": [ + 1836928, + 24576 + ] + }, + { + "name": "isetp64", + "family": "int compare", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; setp.lt.s64 %p0, %d0, %d1; selp.u32 %o0, 1, 0, %p0; setp.lt.u64 %p1, %d0, %d1; selp.u32 %o1, 1, 0, %p1;", + "prog": [ + 1861504, + 928 + ], + "rows": [ + 1862432, + 24576 + ] + }, + { + "name": "plop3", + "family": "int compare", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "setp.ne.u32 %p0, %i0, 0; setp.ne.u32 %p1, %i1, 0; setp.ne.u32 %p2, %i2, 0; xor.pred %p3, %p0, %p1; or.pred %p3, %p3, %p2; not.pred %p3, %p3; selp.u32 %o0, 1, 0, %p3;", + "prog": [ + 1887008, + 800 + ], + "rows": [ + 1887808, + 16384 + ] + }, + { + "name": "shl", + "family": "bits", + "ins": "us", + "outs": "u", + "cases": 1024, + "ptx": "shl.b32 %o0, %i0, %i1;", + "prog": [ + 1904192, + 800 + ], + "rows": [ + 1904992, + 12288 + ] + }, + { + "name": "shr_u32", + "family": "bits", + "ins": "us", + "outs": "u", + "cases": 1024, + "ptx": "shr.u32 %o0, %i0, %i1;", + "prog": [ + 1917280, + 800 + ], + "rows": [ + 1918080, + 12288 + ] + }, + { + "name": "shr_s32", + "family": "bits", + "ins": "us", + "outs": "u", + "cases": 1024, + "ptx": "shr.s32 %o0, %i0, %i1;", + "prog": [ + 1930368, + 800 + ], + "rows": [ + 1931168, + 12288 + ] + }, + { + "name": "shl_imm", + "family": "bits", + "ins": "u", + "outs": "uu", + "cases": 1024, + "ptx": "shl.b32 %o0, %i0, 7; shr.s32 %o1, %i0, 13;", + "prog": [ + 1943456, + 800 + ], + "rows": [ + 1944256, + 12288 + ] + }, + { + "name": "shf_l_wrap", + "family": "bits", + "ins": "uus", + "outs": "u", + "cases": 1024, + "ptx": "shf.l.wrap.b32 %o0, %i0, %i1, %i2;", + "prog": [ + 1956544, + 800 + ], + "rows": [ + 1957344, + 16384 + ] + }, + { + "name": "shf_l_clamp", + "family": "bits", + "ins": "uus", + "outs": "u", + "cases": 1024, + "ptx": "shf.l.clamp.b32 %o0, %i0, %i1, %i2;", + "prog": [ + 1973728, + 800 + ], + "rows": [ + 1974528, + 16384 + ] + }, + { + "name": "shf_r_wrap", + "family": "bits", + "ins": "uus", + "outs": "u", + "cases": 1024, + "ptx": "shf.r.wrap.b32 %o0, %i0, %i1, %i2;", + "prog": [ + 1990912, + 800 + ], + "rows": [ + 1991712, + 16384 + ] + }, + { + "name": "shf_r_clamp", + "family": "bits", + "ins": "uus", + "outs": "u", + "cases": 1024, + "ptx": "shf.r.clamp.b32 %o0, %i0, %i1, %i2;", + "prog": [ + 2008096, + 800 + ], + "rows": [ + 2008896, + 16384 + ] + }, + { + "name": "shl64", + "family": "bits", + "ins": "uus", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; shl.b64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;", + "prog": [ + 2025280, + 800 + ], + "rows": [ + 2026080, + 20480 + ] + }, + { + "name": "shr64_u", + "family": "bits", + "ins": "uus", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; shr.u64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;", + "prog": [ + 2046560, + 800 + ], + "rows": [ + 2047360, + 20480 + ] + }, + { + "name": "shr64_s", + "family": "bits", + "ins": "uus", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; shr.s64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;", + "prog": [ + 2067840, + 800 + ], + "rows": [ + 2068640, + 20480 + ] + }, + { + "name": "lop3", + "family": "bits", + "ins": "uuu", + "outs": "uuuuuuuuuu", + "cases": 1024, + "ptx": "lop3.b32 %o0, %i0, %i1, %i2, 0x96; lop3.b32 %o1, %i0, %i1, %i2, 0xe8; lop3.b32 %o2, %i0, %i1, %i2, 0xca; lop3.b32 %o3, %i0, %i1, %i2, 0x1; lop3.b32 %o4, %i0, %i1, %i2, 0x80; lop3.b32 %o5, %i0, %i1, %i2, 0x6a; lop3.b32 %o6, %i0, %i1, %i2, 0xf8; lop3.b32 %o7, %i0, %i1, %i2, 0x1e; lop3.b32 %o8, %i0, %i1, %i2, 0xb4; lop3.b32 %o9, %i0, %i1, %i2, 0x3c;", + "prog": [ + 2089120, + 1056 + ], + "rows": [ + 2090176, + 53248 + ] + }, + { + "name": "logic", + "family": "bits", + "ins": "uu", + "outs": "uuuu", + "cases": 1024, + "ptx": "and.b32 %o0, %i0, %i1; or.b32 %o1, %i0, %i1; xor.b32 %o2, %i0, %i1; not.b32 %t0, %i0; and.b32 %o3, %t0, %i1;", + "prog": [ + 2143424, + 928 + ], + "rows": [ + 2144352, + 24576 + ] + }, + { + "name": "prmt", + "family": "bits", + "ins": "uuu", + "outs": "uuuuuuu", + "cases": 1024, + "ptx": "prmt.b32 %o0, %i0, %i1, %i2; prmt.b32.f4e %o1, %i0, %i1, %i2; prmt.b32.b4e %o2, %i0, %i1, %i2; prmt.b32.rc8 %o3, %i0, %i1, %i2; prmt.b32.ecl %o4, %i0, %i1, %i2; prmt.b32.ecr %o5, %i0, %i1, %i2; prmt.b32.rc16 %o6, %i0, %i1, %i2;", + "prog": [ + 2168928, + 928 + ], + "rows": [ + 2169856, + 40960 + ] + }, + { + "name": "prmt_imm", + "family": "bits", + "ins": "uu", + "outs": "uuu", + "cases": 1024, + "ptx": "prmt.b32 %o0, %i0, %i1, 0x3210; prmt.b32 %o1, %i0, %i1, 0x5140; prmt.b32 %o2, %i0, %i1, 0x8880;", + "prog": [ + 2210816, + 800 + ], + "rows": [ + 2211616, + 20480 + ] + }, + { + "name": "bfe_u32", + "family": "bits", + "ins": "uss", + "outs": "u", + "cases": 1024, + "ptx": "bfe.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 2232096, + 800 + ], + "rows": [ + 2232896, + 16384 + ] + }, + { + "name": "bfe_s32", + "family": "bits", + "ins": "uss", + "outs": "u", + "cases": 1024, + "ptx": "bfe.s32 %o0, %i0, %i1, %i2;", + "prog": [ + 2249280, + 800 + ], + "rows": [ + 2250080, + 16384 + ] + }, + { + "name": "bfi", + "family": "bits", + "ins": "uuss", + "outs": "u", + "cases": 1024, + "ptx": "bfi.b32 %o0, %i0, %i1, %i2, %i3;", + "prog": [ + 2266464, + 928 + ], + "rows": [ + 2267392, + 20480 + ] + }, + { + "name": "bmsk_clamp", + "family": "bits", + "ins": "ss", + "outs": "u", + "cases": 1024, + "ptx": "bmsk.clamp.b32 %o0, %i0, %i1;", + "prog": [ + 2287872, + 800 + ], + "rows": [ + 2288672, + 12288 + ] + }, + { + "name": "bmsk_wrap", + "family": "bits", + "ins": "ss", + "outs": "u", + "cases": 1024, + "ptx": "bmsk.wrap.b32 %o0, %i0, %i1;", + "prog": [ + 2300960, + 800 + ], + "rows": [ + 2301760, + 12288 + ] + }, + { + "name": "szext_s", + "family": "bits", + "ins": "us", + "outs": "u", + "cases": 1024, + "ptx": "szext.clamp.s32 %o0, %i0, %i1;", + "prog": [ + 2314048, + 800 + ], + "rows": [ + 2314848, + 12288 + ] + }, + { + "name": "popc", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "popc.b32 %o0, %i0;", + "prog": [ + 2327136, + 800 + ], + "rows": [ + 2327936, + 8192 + ] + }, + { + "name": "popc64", + "family": "bits", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; popc.b64 %o0, %d0;", + "prog": [ + 2336128, + 800 + ], + "rows": [ + 2336928, + 12288 + ] + }, + { + "name": "clz", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "clz.b32 %o0, %i0;", + "prog": [ + 2349216, + 800 + ], + "rows": [ + 2350016, + 8192 + ] + }, + { + "name": "clz64", + "family": "bits", + "ins": "uu", + "outs": "u", + "cases": 1024, + "ptx": "mov.b64 %d0, {%i0, %i1}; clz.b64 %o0, %d0;", + "prog": [ + 2358208, + 800 + ], + "rows": [ + 2359008, + 12288 + ] + }, + { + "name": "bfind_u32", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "bfind.u32 %o0, %i0;", + "prog": [ + 2371296, + 800 + ], + "rows": [ + 2372096, + 8192 + ] + }, + { + "name": "bfind_s32", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "bfind.s32 %o0, %i0;", + "prog": [ + 2380288, + 800 + ], + "rows": [ + 2381088, + 8192 + ] + }, + { + "name": "bfind_shiftamt", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "bfind.shiftamt.u32 %o0, %i0;", + "prog": [ + 2389280, + 800 + ], + "rows": [ + 2390080, + 8192 + ] + }, + { + "name": "brev", + "family": "bits", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "brev.b32 %o0, %i0;", + "prog": [ + 2398272, + 800 + ], + "rows": [ + 2399072, + 8192 + ] + }, + { + "name": "fns", + "family": "bits", + "ins": "uus", + "outs": "u", + "cases": 1024, + "ptx": "and.b32 %t0, %i2, 31; fns.b32 %o0, %i0, %i1, %t0;", + "prog": [ + 2407264, + 1696 + ], + "rows": [ + 2408960, + 16384 + ] + }, + { + "name": "dp4a_u32", + "family": "bits", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "dp4a.u32.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 2425344, + 800 + ], + "rows": [ + 2426144, + 16384 + ] + }, + { + "name": "dp4a_s32", + "family": "bits", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "dp4a.s32.s32 %o0, %i0, %i1, %i2;", + "prog": [ + 2442528, + 800 + ], + "rows": [ + 2443328, + 16384 + ] + }, + { + "name": "dp4a_su", + "family": "bits", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "dp4a.s32.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 2459712, + 800 + ], + "rows": [ + 2460512, + 16384 + ] + }, + { + "name": "dp2a_lo", + "family": "bits", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "dp2a.lo.s32.s32 %o0, %i0, %i1, %i2;", + "prog": [ + 2476896, + 800 + ], + "rows": [ + 2477696, + 16384 + ] + }, + { + "name": "dp2a_hi", + "family": "bits", + "ins": "uuu", + "outs": "u", + "cases": 1024, + "ptx": "dp2a.hi.u32.u32 %o0, %i0, %i1, %i2;", + "prog": [ + 2494080, + 800 + ], + "rows": [ + 2494880, + 16384 + ] + }, + { + "name": "shfl_idx", + "family": "warp", + "ins": "ul", + "outs": "u", + "cases": 1024, + "ptx": "shfl.sync.idx.b32 %o0, %i0, %i1, 0x1f, 0xffffffff;", + "prog": [ + 2511264, + 800 + ], + "rows": [ + 2512064, + 12288 + ] + }, + { + "name": "shfl_up", + "family": "warp", + "ins": "us", + "outs": "uu", + "cases": 1024, + "ptx": "and.b32 %t0, %i1, 31; shfl.sync.up.b32 %o0|%p0, %i0, %t0, 0, 0xffffffff; selp.u32 %o1, 1, 0, %p0;", + "prog": [ + 2524352, + 800 + ], + "rows": [ + 2525152, + 16384 + ] + }, + { + "name": "shfl_down", + "family": "warp", + "ins": "us", + "outs": "uu", + "cases": 1024, + "ptx": "and.b32 %t0, %i1, 31; shfl.sync.down.b32 %o0|%p0, %i0, %t0, 0x1f, 0xffffffff; selp.u32 %o1, 1, 0, %p0;", + "prog": [ + 2541536, + 800 + ], + "rows": [ + 2542336, + 16384 + ] + }, + { + "name": "shfl_bfly", + "family": "warp", + "ins": "ul", + "outs": "u", + "cases": 1024, + "ptx": "shfl.sync.bfly.b32 %o0, %i0, %i1, 0x1f, 0xffffffff;", + "prog": [ + 2558720, + 800 + ], + "rows": [ + 2559520, + 12288 + ] + }, + { + "name": "shfl_bfly_imm", + "family": "warp", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "shfl.sync.bfly.b32 %o0, %i0, 1, 0x1f, 0xffffffff;", + "prog": [ + 2571808, + 800 + ], + "rows": [ + 2572608, + 8192 + ] + }, + { + "name": "shfl_idx_seg", + "family": "warp", + "ins": "ul", + "outs": "u", + "cases": 1024, + "ptx": "shfl.sync.idx.b32 %o0, %i0, %i1, 0x1807, 0xffffffff;", + "prog": [ + 2580800, + 800 + ], + "rows": [ + 2581600, + 12288 + ] + }, + { + "name": "vote", + "family": "warp", + "ins": "p", + "outs": "uuuu", + "cases": 1024, + "ptx": "and.b32 %t0, %i0, 1; setp.ne.u32 %p0, %t0, 0; vote.sync.ballot.b32 %o0, %p0, 0xffffffff; vote.sync.any.pred %p1, %p0, 0xffffffff; selp.u32 %o1, 1, 0, %p1; vote.sync.all.pred %p2, %p0, 0xffffffff; selp.u32 %o2, 1, 0, %p2; vote.sync.uni.pred %p3, %p0, 0xffffffff; selp.u32 %o3, 1, 0, %p3;", + "prog": [ + 2593888, + 928 + ], + "rows": [ + 2594816, + 20480 + ] + }, + { + "name": "activemask", + "family": "warp", + "ins": "p", + "outs": "u", + "cases": 1024, + "ptx": "activemask.b32 %o0;", + "prog": [ + 2615296, + 800 + ], + "rows": [ + 2616096, + 8192 + ] + }, + { + "name": "match_any", + "family": "warp", + "ins": "p", + "outs": "u", + "cases": 1024, + "ptx": "and.b32 %t0, %i0, 3; match.any.sync.b32 %o0, %t0, 0xffffffff;", + "prog": [ + 2624288, + 800 + ], + "rows": [ + 2625088, + 8192 + ] + }, + { + "name": "redux_add", + "family": "warp", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "redux.sync.add.u32 %o0, %i0, 0xffffffff;", + "prog": [ + 2633280, + 800 + ], + "rows": [ + 2634080, + 8192 + ] + }, + { + "name": "redux_min_s32", + "family": "warp", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "redux.sync.min.s32 %o0, %i0, 0xffffffff;", + "prog": [ + 2642272, + 800 + ], + "rows": [ + 2643072, + 8192 + ] + }, + { + "name": "redux_or", + "family": "warp", + "ins": "u", + "outs": "u", + "cases": 1024, + "ptx": "redux.sync.or.b32 %o0, %i0, 0xffffffff;", + "prog": [ + 2651264, + 800 + ], + "rows": [ + 2652064, + 8192 + ] + }, + { + "name": "hmma_16816_f16", + "family": "tensor", + "ins": "HHHHHHHH", + "outs": "hh", + "cases": 1024, + "ptx": "mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%o0, %o1}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%i6, %i7};", + "prog": [ + 2660256, + 928 + ], + "rows": [ + 2661184, + 40960 + ] + }, + { + "name": "hmma_16816_f32", + "family": "tensor", + "ins": "HHHHHHFFFF", + "outs": "ffff", + "cases": 1024, + "ptx": "mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;", + "prog": [ + 2702144, + 928 + ], + "rows": [ + 2703072, + 57344 + ] + }, + { + "name": "hmma_1688_f16", + "family": "tensor", + "ins": "HHHHH", + "outs": "hh", + "cases": 1024, + "ptx": "mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%o0, %o1}, {%i0, %i1}, {%i2}, {%i3, %i4};", + "prog": [ + 2760416, + 800 + ], + "rows": [ + 2761216, + 28672 + ] + }, + { + "name": "hmma_16816_bf16", + "family": "tensor", + "ins": "BBBBBBFFFF", + "outs": "ffff", + "cases": 1024, + "ptx": "mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;", + "prog": [ + 2789888, + 928 + ], + "rows": [ + 2790816, + 57344 + ] + }, + { + "name": "hmma_1688_tf32", + "family": "tensor", + "ins": "FFFFFFFFFF", + "outs": "ffff", + "cases": 1024, + "ptx": "mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;", + "prog": [ + 2848160, + 928 + ], + "rows": [ + 2849088, + 57344 + ] + }, + { + "name": "imma_16832_s8", + "family": "tensor", + "ins": "UUUUUUUUUU", + "outs": "uuuu", + "cases": 1024, + "ptx": "mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%o0, %o1, %o2, %o3}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%i6, %i7, %i8, %i9};", + "prog": [ + 2906432, + 928 + ], + "rows": [ + 2907360, + 57344 + ] + }, + { + "name": "dadd", + "family": "f64", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %g0, {%i0, %i1}; mov.b64 %g1, {%i2, %i3}; add.rn.f64 %g2, %g0, %g1; mov.b64 {%o0, %o1}, %g2;", + "prog": [ + 2964704, + 800 + ], + "rows": [ + 2965504, + 24576 + ] + }, + { + "name": "dmul", + "family": "f64", + "ins": "uuuu", + "outs": "uu", + "cases": 1024, + "ptx": "mov.b64 %g0, {%i0, %i1}; mov.b64 %g1, {%i2, %i3}; mul.rn.f64 %g2, %g0, %g1; mov.b64 {%o0, %o1}, %g2;", + "prog": [ + 2990080, + 800 + ], + "rows": [ + 2990880, + 24576 + ] + } + ] +} diff --git a/dm/gfx/tests/test_dmsh_ampere_orin.py b/dm/gfx/tests/test_dmsh_ampere_orin.py new file mode 100644 index 0000000000..e8d7c0557d --- /dev/null +++ b/dm/gfx/tests/test_dmsh_ampere_orin.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +"""Ampere instructions through the whole chain against a real Ampere GPU: each op of ampere_orin/ (recorded on a Jetson Orin, sm_87, by +dm/gfx/tools/ampere_orin/capture.py) is the GPU compiler's machine code for one PTX operation, its 1,024 input rows and the outputs the GPU computed. Here the +program goes through the translator (dmsh_xlate, arch sm86, the build's debug and relaxed-precision options), spirv-cross with the chain's arguments and the +MSL rewrites dmsh_build.py's finish() makes of every shader, is compiled with the chain's Metal flags and run on this Mac's GPU on the same inputs, and every +output word is compared with the GPU's bit for bit. + +An op is expected to match except for what Metal's float arithmetic itself gives (compare.PLATFORM: NaN bits, denormals). KNOWN lists the ops that still +differ, each with its reason and where it is tracked; APPROX the ones that run Metal's own approximation of a function (bit-exact only by accident). The test +fails when an op outside those lists differs, and when a KNOWN op matches, so that its entry is taken out with the fix. + +Needs DMSH_TEST_XLATE (the translator) like the other translator tests, and an Apple-silicon Mac. AMPERE_ORIN_ONLY= runs some; +AMPERE_ORIN_REPORT= writes every op's causes as JSON.""" +import ast +import bisect +import concurrent.futures +import gzip +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +TOOLS = HERE.parent / 'tools' +FIXTURE = HERE / 'ampere_orin' +sys.path.insert(0, str(TOOLS)) +sys.path.insert(0, str(TOOLS / 'ampere_orin')) +import compare # noqa: E402 + +# Ops that still differ from the GPU beyond PLATFORM: name -> why. A fix takes its ops out of here. +KNOWN = { + # results that differ + 'fadd_rz': 'FADD .RZ / .RM / .RP: the rounding mode is dropped, Metal rounds to nearest', 'fadd_rm': '(as fadd_rz)', 'fadd_rp': '(as fadd_rz)', + 'fmul_rz': 'FMUL .RZ / .RM / .RP: the rounding mode is dropped', 'fmul_rm': '(as fmul_rz)', 'fmul_rp': '(as fmul_rz)', + 'ffma_rz': 'FFMA .RZ / .RM / .RP: the rounding mode is dropped', 'ffma_rm': '(as ffma_rz)', 'ffma_rp': '(as ffma_rz)', + 'i2f_s32_rz': 'I2FP.F32.S32.RZ rounds to nearest (PR #1800 has the fix)', + 'fmax_nan': 'FMNMX.NAN: a NaN operand gives the other operand, not NaN (PR #1800 has the fix)', 'fmin_nan': '(as fmax_nan)', + 'fsetp': 'FSETP.NE (ordered) becomes MSL !=, which is unordered: true with a NaN operand', 'fsetp_ftz': '(as fsetp)', + 'shfl_idx': 'SHFL: a lane in a register is not masked to its low 5 bits', 'shfl_bfly': '(as shfl_idx)', 'shfl_idx_seg': '(as shfl_idx)', + 'f2fp_pack_bf16x2': 'F2FP.BF16.PACK_AB packs f16, not bf16', 'f2fp_pack_f16x2_relu': 'F2FP.RELU: negative results are not clamped to 0', + 'i2f_s64': 'I2F.S64 rounds twice (the high and low words apart)', + 'fmax': 'FMNMX: Ampere orders -0 below +0 (max(+0, -0) is +0); Metal returns either', 'hmax2': '(as fmax, HMNMX2)', + 'hadd2_ftz': 'HADD2 / HMUL2 / HFMA2 .FTZ: f16 denormals are not flushed (Metal keeps them)', 'hmul2_ftz': '(as hadd2_ftz)', + 'hfma2_ftz': '(as hadd2_ftz)', + 'hmma_16816_f32': 'HMMA.16816.F32: the f32 accumulator does not add the products as the tensor core does (about 1 ULP)', + # the translator refuses these + 'fdiv_rn': 'FCHK (0x302) is not implemented; its slow path is a CALL.ABS into the CUDA driver (as frcp_rn)', + 'f2i_s64': 'F2I.S64 (signed 64-bit, 0x311) is not implemented: only the unsigned truncation of an F32 is', + 'dadd': 'DADD (0x229) is not implemented', 'dmul': 'DMUL (0x228) is not implemented', + 'hmin2_nan': 'HMNMX2.NAN is not implemented', 'bmin2': 'HMNMX2.BF16_V2 is not implemented', 'bmax2_nan': '(as bmin2)', + 'bfma2': 'HFMA2.BF16_V2 is not implemented', 'hmma_16816_bf16': 'HMMA .BF16 is not implemented', 'hmma_1688_tf32': 'HMMA .TF32 is not implemented', + 'imma_16832_s8': 'IMMA is not implemented', 'dp4a_u32': 'IDP.4A is not implemented', 'dp4a_s32': '(as dp4a_u32)', 'dp4a_su': '(as dp4a_u32)', + 'dp2a_lo': 'IDP.2A is not implemented', 'dp2a_hi': '(as dp2a_lo)', 'i2i_s8_sat': 'I2I .SAT is not implemented', 'i2i_u16_sat': '(as i2i_s8_sat)', + 'isad': 'VABSDIFF is not implemented', 'iadd_sat': 'PLOP3 with .SIGN register sources (0x21f) is not implemented', 'isub_sat': '(as iadd_sat)', + 'prmt': 'PRMT modes other than the index form (.F4E .B4E .RC8 .ECL .ECR .RC16) are not implemented', + 'mufu_tanh': 'MUFU.TANH is not implemented', 'mufu_tanh_f16x2': '(as mufu_tanh)', + # the program calls a routine of the CUDA driver's own (CALL.ABS to a relocation the JIT resolves at load), which a game's shader never does + 'frcp_rn': 'not testable: CALL.ABS into the CUDA driver', 'fsqrt_rn': '(as frcp_rn)', +} +# Ops that run Metal's own approximation (MUFU, and the divisions built on it): compared and reported, never failed +APPROX = {'mufu_ex2', 'mufu_ex2_ftz', 'mufu_lg2', 'mufu_lg2_ftz', 'mufu_rcp', 'mufu_rcp_ftz', 'mufu_rsqrt', 'mufu_rsqrt_ftz', 'mufu_sqrt', 'mufu_sqrt_ftz', 'mufu_sin', + 'mufu_cos', 'mufu_ex2_f16x2', 'fdiv_full', 'fdiv_approx', + # HMMA with an f16 accumulator: one block of 8 rounded once (PR #1897) matches 99.97 %; the rest are 1-2 ULP where the sum nearly cancels + 'hmma_16816_f16', 'hmma_1688_f16'} + +MAC_METAL = ['xcrun', '-sdk', 'macosx', 'metal', '-c', '-O1', '-fpreserve-invariance', '-fwrapv'] # dmsh_build.py MAC_METAL + + +def load_fixture(): + doc = json.loads((FIXTURE / 'manifest.json').read_text()) + blob = gzip.decompress((FIXTURE / 'ampere_orin.u32.gz').read_bytes()) + import array + for o in doc['ops']: + o['program'] = blob[o['prog'][0]:o['prog'][0] + o['prog'][1]] + words = array.array('I') + words.frombytes(blob[o['rows'][0]:o['rows'][0] + o['rows'][1]]) + if sys.byteorder != 'little': + words.byteswap() + w = len(o['ins']) + len(o['outs']) + o['rows_in'] = [list(words[i:i + len(o['ins'])]) for i in range(0, len(words), w)] + o['rows_gpu'] = [list(words[i + len(o['ins']):i + w]) for i in range(0, len(words), w)] + return doc + + +def chain_rewrites(): + """the MSL rewrites dmsh_build.py's finish() makes of every shader, in its order: each `src = f(src)` at its top level or in a try there (the ones under a + stage's `if` are another stage's). dmsh_build.py runs as a script, so its functions and constants are taken from its source, as test_dmsh_hmma.py does""" + import dmsh_atomic + import dmsh_wide + path = TOOLS / 'dmsh_build.py' + tree = ast.parse(path.read_text()) + scope = {'re': re, 'bisect': bisect, 'os': os, 'json': json, 'dmsh_atomic': dmsh_atomic, 'dmsh_wide': dmsh_wide} + for n in tree.body: + pure = isinstance(n, ast.Assign) and all(isinstance(t, ast.Name) for t in n.targets) and not any( + isinstance(c, ast.Call) and getattr(c.func, 'attr', '') != 'compile' for c in ast.walk(n.value)) + if isinstance(n, ast.FunctionDef) or pure: + try: + exec(compile(ast.Module(body=[n], type_ignores=[]), str(path), 'exec'), scope) + except Exception: # a constant built from what only a build has (the game, the chain version) is not needed here + pass + finish = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'finish') + steps = [] + + def visit(body): + for st in body: + if isinstance(st, ast.Try): + visit(st.body) + elif (isinstance(st, ast.Assign) and len(st.targets) == 1 and getattr(st.targets[0], 'id', None) == 'src' and isinstance(st.value, ast.Call) + and len(st.value.args) == 1 and getattr(st.value.args[0], 'id', None) == 'src' and not st.value.keywords): + steps.append(ast.unparse(st.value.func)) + visit(finish.body) + fns = [] + for name in steps: + mod, _, attr = name.rpartition('.') + fns.append((name, getattr(scope[mod], attr) if mod else scope[name])) + return fns, scope['XC'] + + +def harness(msl, w): + """main0 seeds each thread's W words of the shader's shared array from buffer 30 and writes them back where it returns""" + i = msl.index('kernel void main0(') + sig = msl[i:msl.index(')', i)] + lid = re.search(r'uint (\w+) \[\[thread_index_in_threadgroup\]\]', sig) + lid3 = re.search(r'uint3 (\w+) \[\[thread_position_in_threadgroup\]\]', sig) + tg = re.search(r'uint3 (\w+) \[\[threadgroup_position_in_grid\]\]', sig) + add = ['device uint* dm_io [[buffer(30)]]'] + if lid: + lid_e = lid.group(1) + elif lid3: + lid_e = lid3.group(1) + '.x' + else: + add.append('uint dm_lid [[thread_index_in_threadgroup]]') + lid_e = 'dm_lid' + if tg: + tg_e = tg.group(1) + '.x' + else: + add.append('uint3 dm_tg [[threadgroup_position_in_grid]]') + tg_e = 'dm_tg.x' + rest = msl[i + len('kernel void main0('):] + msl = msl[:i] + 'kernel void main0(' + ', '.join(add) + (', ' if not rest.lstrip().startswith(')') else '') + rest + start = msl.index('{', msl.index(')', msl.index('kernel void main0('))) + 1 + end = msl.rindex('}') + body = msl[start:end] + shm = re.search(r'threadgroup spvUnsafeArray (\w+);|threadgroup uint (\w+)\[(\d+)\];', body) + if not shm: + raise ValueError('no shared array in main0') + arr, words = shm.group(2) or shm.group(3), int(shm.group(1) or shm.group(4)) + if words < 32 * w: + raise ValueError(f'shared array of {words} words, {32 * w} needed') + base = f'(({tg_e}) * 32u + ({lid_e})) * {w}u' + dump = f'for (uint dm_k = 0; dm_k < {w}u; dm_k++) dm_io[{base} + dm_k] = {arr}[({lid_e}) * {w}u + dm_k];' + body = (body[:shm.end()] + f'\n for (uint dm_k = 0; dm_k < {w}u; dm_k++) {arr}[({lid_e}) * {w}u + dm_k] = dm_io[{base} + dm_k];\n' + + body[shm.end():]) + body = re.sub(r'\breturn;', '{ ' + dump + ' return; }', body).rstrip() + '\n ' + dump + '\n' + return msl[:start] + body + msl[end:] + + +RUNNER = r'''#import +#import +// run : main0 over groups of 32 threads, io at buffer 30 (the other slots a zero buffer), written back over io +int main(int argc, char **argv) { @autoreleasepool { + id d = MTLCreateSystemDefaultDevice(); NSError *e = nil; + if (!d) { fprintf(stderr, "no Metal device\n"); return 1; } + id lib = [d newLibraryWithURL:[NSURL fileURLWithPath:@(argv[1])] error:&e]; if (!lib) { fprintf(stderr, "library: %s\n", e.description.UTF8String); return 1; } + id f = [lib newFunctionWithName:@"main0"]; if (!f) { fprintf(stderr, "no main0\n"); return 1; } + id ps = [d newComputePipelineStateWithFunction:f error:&e]; if (!ps) { fprintf(stderr, "pipeline: %s\n", e.description.UTF8String); return 1; } + NSData *in = [NSData dataWithContentsOfFile:@(argv[2])]; if (!in) { fprintf(stderr, "no input\n"); return 1; } + id b = [d newBufferWithBytes:in.bytes length:in.length options:MTLResourceStorageModeShared]; + id zero = [d newBufferWithLength:65536 options:MTLResourceStorageModeShared]; + id q = [d newCommandQueue]; id cb = [q commandBuffer]; id ce = [cb computeCommandEncoder]; + if (!q || !cb || !ce || !b || !zero) { fprintf(stderr, "Metal objects\n"); return 1; } + [ce setComputePipelineState:ps]; + for (int i = 0; i < 30; i++) [ce setBuffer:zero offset:0 atIndex:i]; + [ce setBuffer:b offset:0 atIndex:30]; + [ce dispatchThreadgroups:MTLSizeMake(atoi(argv[3]), 1, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)]; + [ce endEncoding]; [cb commit]; [cb waitUntilCompleted]; + if (cb.error) { fprintf(stderr, "gpu: %s\n", cb.error.description.UTF8String); return 1; } + [[NSData dataWithBytes:b.contents length:in.length] writeToFile:@(argv[2]) atomically:YES]; return 0; } } +''' + + +def run_op(o, work, runner, steps, xc, env): + """('ok', metal rows) or (stage that failed, its message)""" + import array + import dmsh_atomic + import dmsh_geometry + d = work / o['name'] + d.mkdir() + (d / 'k.bin').write_bytes(o['program']) + w = len(o['ins']) + len(o['outs']) + job = [dict(key='k', stage='cs', arch='sm86', code=str(d / 'k.bin'), tex={}, img={}, attr={}, fout={}, local=[32, 1, 1], debug=True, reduced=True, + nvn2_cs=[0, 0, 3, 0x40, 0], shared=32 * w * 4)] + (d / 'jobs.json').write_text(json.dumps(job)) + r = subprocess.run([os.environ['DMSH_TEST_XLATE'], str(d / 'jobs.json'), str(d)], capture_output=True, text=True, env=env) + if r.returncode or not (d / 'k.spv').exists(): + return 'translator', next((ln for ln in (r.stdout + r.stderr).splitlines() if '[RED]' in ln), (r.stdout + r.stderr).strip()[-300:]) + dmsh_geometry.flatten_sets(d / 'k.spv') + r = subprocess.run([str(TOOLS / 'spirv_cross_bin/spirv-cross'), *dmsh_atomic.cross_args(xc, str(d / 'k.spv')), str(d / 'k.spv')], capture_output=True, text=True) + if r.returncode: + return 'spirv-cross', r.stderr.strip()[-300:] + src = r.stdout + try: + for _, fn in steps: + src = fn(src) + src = harness(src, w) + except Exception as e: + return 'MSL rewrite', f'{type(e).__name__}: {e}' + (d / 'k.metal').write_text(src) + r = subprocess.run(MAC_METAL + [str(d / 'k.metal'), '-o', str(d / 'k.air')], capture_output=True, text=True) + if r.returncode: + return 'Metal compile', r.stderr.strip()[-300:] + subprocess.run(['xcrun', '-sdk', 'macosx', 'metallib', str(d / 'k.air'), '-o', str(d / 'k.metallib')], check=True, capture_output=True) + io = array.array('I', [0xcdcdcdcd]) * (len(o['rows_in']) * w) + for c, row in enumerate(o['rows_in']): + io[c * w:c * w + len(row)] = array.array('I', row) + (d / 'io.bin').write_bytes(io.tobytes()) + r = subprocess.run([str(runner), str(d / 'k.metallib'), str(d / 'io.bin'), str(len(o['rows_in']) // 32)], capture_output=True, text=True, timeout=120) + if r.returncode: + return 'Metal run', r.stderr.strip()[-300:] + out = array.array('I') + out.frombytes((d / 'io.bin').read_bytes()) + return 'ok', [list(out[c * w + len(o['ins']):(c + 1) * w]) for c in range(len(o['rows_in']))] + + +@unittest.skipUnless(os.environ.get('DMSH_TEST_XLATE'), 'set DMSH_TEST_XLATE to the offline translator') +@unittest.skipUnless(sys.platform == 'darwin' and shutil.which('xcrun'), 'needs a Mac with Xcode') +class AgainstAmpere(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.doc = load_fixture() + only = set(filter(None, os.environ.get('AMPERE_ORIN_ONLY', '').split(','))) + ops = [o for o in cls.doc['ops'] if not only or o['name'] in only or o['family'] in only] + steps, xc = chain_rewrites() + cls.tmp = tempfile.TemporaryDirectory(prefix='ampere_orin_') + work = Path(cls.tmp.name) + (work / 'run.m').write_text(RUNNER) + subprocess.run(['clang', '-fobjc-arc', '-O1', str(work / 'run.m'), '-framework', 'Metal', '-framework', 'Foundation', '-o', str(work / 'run')], check=True) + env = dict(os.environ) + with concurrent.futures.ThreadPoolExecutor(max(2, (os.cpu_count() or 4) - 2)) as ex: + futs = {o['name']: ex.submit(run_op, o, work, work / 'run', steps, xc, env) for o in ops} + cls.results = {} + for o in ops: + stage, got = futs[o['name']].result() + if stage == 'ok': + causes, lanes, worst, examples = compare.classify(o, o['rows_in'], o['rows_gpu'], got) + cls.results[o['name']] = dict(stage='ok', lanes=lanes, causes=dict(causes), worst_ulp=worst, examples=examples) + else: + cls.results[o['name']] = dict(stage=stage, message=got) + cls.ops = ops + if os.environ.get('AMPERE_ORIN_REPORT'): + Path(os.environ['AMPERE_ORIN_REPORT']).write_text(json.dumps({'device': cls.doc['device'], 'rewrites': [n for n, _ in steps], 'ops': cls.results}, indent=1)) + cls.print_summary() + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + @classmethod + def print_summary(cls): + lines = [f"\nAmpere ops against the {cls.doc['device'].get('gpu', 'GPU')} (sm_{cls.doc['device'].get('sm', '?')}, CUDA driver " + f"{cls.doc['device'].get('cuda_driver', '?')}): lanes that differ, by cause"] + for o in cls.ops: + r = cls.results[o['name']] + tag = 'known' if o['name'] in KNOWN else 'approx' if o['name'] in APPROX else '' + if r['stage'] != 'ok': + lines.append(f" {o['name']:22s} {tag:6s} {r['stage']} failed: {r['message'][:120]}") + continue + diff = sum(r['causes'].values()) + cs = ', '.join(f'{k} {v}' for k, v in sorted(r['causes'].items(), key=lambda kv: -kv[1])) + lines.append(f" {o['name']:22s} {tag:6s} {diff:5d} of {r['lanes']:6d}" + (f' ({cs})' if cs else '') + (f" max {r['worst_ulp']} ULP" if r['worst_ulp'] else '')) + print('\n'.join(lines), file=sys.stderr) + + def test_matches_the_gpu(self): + for o in self.ops: + n, r = o['name'], self.results[o['name']] + with self.subTest(op=n): + beyond = {k: v for k, v in r.get('causes', {}).items() if k not in compare.PLATFORM} + if n in APPROX: + self.assertEqual(r['stage'], 'ok', r.get('message')) + elif n in KNOWN: + self.assertTrue(r['stage'] != 'ok' or beyond, f'{n} now matches the GPU: take it out of KNOWN ({KNOWN[n]})') + else: + self.assertEqual(r['stage'], 'ok', f"{n}: {r['stage']} failed: {r.get('message')}") + self.assertFalse(beyond, f"{n} differs from the GPU: {beyond}; for example {r['examples'][:2]}") + + def test_every_listed_op_is_recorded(self): # a KNOWN or APPROX entry for an op the fixture lacks is a typo + names = {o['name'] for o in self.doc['ops']} + self.assertFalse((set(KNOWN) | APPROX) - names) + + +if __name__ == '__main__': + unittest.main() diff --git a/dm/gfx/tools/ampere_orin/batch.py b/dm/gfx/tools/ampere_orin/batch.py new file mode 100644 index 0000000000..0c789f8cf7 --- /dev/null +++ b/dm/gfx/tools/ampere_orin/batch.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""On the Jetson: for every op in manifest.json, JIT its PTX to a cubin, disassemble it with nvdisasm, and run it on the op's inputs. + + batch.py manifest.json (run from the folder holding the ops' .ptx and .in.bin files) + +Writes .cubin, .sass and .orin.bin beside them, and .err when a step fails. Stdlib only, through cu.py.""" +import json +import os +import subprocess +import sys +import traceback + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import cu # noqa: E402 +import prog # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def main(): + import ctypes as C + cu.cu.cuGetErrorString.argtypes = [C.c_int, C.POINTER(C.c_char_p)] + cu.cu.cuLinkAddData_v2.argtypes = [C.c_void_p, C.c_int, C.c_void_p, C.c_size_t, C.c_char_p, C.c_uint, C.c_void_p, C.c_void_p] + cu.cu.cuMemcpyHtoD_v2.argtypes = [C.c_uint64, C.c_char_p, C.c_size_t] + cu.cu.cuMemcpyDtoH_v2.argtypes = [C.c_void_p, C.c_uint64, C.c_size_t] + cu.cu.cuMemsetD8_v2.argtypes = [C.c_uint64, C.c_ubyte, C.c_size_t] + cu.cu.cuMemFree_v2.argtypes = [C.c_uint64] + cu.cu.cuLaunchKernel.argtypes = [C.c_void_p] + [C.c_uint] * 7 + [C.c_void_p, C.c_void_p, C.c_void_p] + cu.init() + ops = json.load(open(sys.argv[1])) + ok = 0 + name, drv = C.create_string_buffer(256), C.c_int() + cu.cu.cuDeviceGetName(name, 256, 0) + cu.cu.cuDriverGetVersion(C.byref(drv)) + maj, mnr = C.c_int(), C.c_int() + cu.cu.cuDeviceGetAttribute(C.byref(maj), 75, 0) # CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR + cu.cu.cuDeviceGetAttribute(C.byref(mnr), 76, 0) # ..._MINOR + nvd = subprocess.run([os.path.join(HERE, 'nvdisasm'), '--version'], capture_output=True, text=True).stdout.strip().splitlines()[-1:] + json.dump({'gpu': name.value.decode(), 'sm': f'{maj.value}{mnr.value}', 'cuda_driver': f'{drv.value // 1000}.{drv.value % 1000 // 10}', + 'nvdisasm': nvd[0] if nvd else ''}, open('device.info', 'w')) + for op in ops: + n = op['name'] + for ext in ('.err', '.orin.bin', '.prog.bin', '.prog.sass'): + if os.path.exists(n + ext): + os.remove(n + ext) + try: + cubin = cu.ptx2cubin(open(n + '.ptx', 'rb').read()) + open(n + '.cubin', 'wb').write(cubin) + sass = subprocess.run([os.path.join(HERE, 'nvdisasm'), '-hex', '-c', n + '.cubin'], capture_output=True, text=True, check=True).stdout + open(n + '.sass', 'w').write(sass) + out = cu.run(cubin, 'k', op['groups'], 32, 0, [open(n + '.in.bin', 'rb').read()], op['groups'] * 32 * op['nout'] * 4) + open(n + '.orin.bin', 'wb').write(out) + code, rb, notes = prog.rewrite(cubin, sass, op['nin'], op['nout']) # the Metal side's program, checked by the disassembler here + open(n + '.prog.bin', 'wb').write(code) + dis = subprocess.run([os.path.join(HERE, 'nvdisasm'), '--binary', 'SM87', '-hex', n + '.prog.bin'], capture_output=True, text=True, check=True).stdout + open(n + '.prog.sass', 'w').write(f'// base register R{rb}\n' + ''.join(f'// {x}\n' for x in notes) + dis) + ok += 1 + except BaseException as e: # cu.ck raises SystemExit + open(n + '.err', 'w').write(f'{type(e).__name__}: {e}\n{traceback.format_exc()}') + print(f'{n}: {e}', file=sys.stderr) + if 'kernel' in str(e) or 'cuLaunchKernel' in str(e): # a fault poisons the context: start a fresh one + cu.init() + print(f'{ok}/{len(ops)} ops ran') + + +if __name__ == '__main__': + main() diff --git a/dm/gfx/tools/ampere_orin/capture.py b/dm/gfx/tools/ampere_orin/capture.py new file mode 100644 index 0000000000..3307775e72 --- /dev/null +++ b/dm/gfx/tools/ampere_orin/capture.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Records the fixture of dm/gfx/tests/test_dmsh_ampere_orin.py on a Linux machine with an Ampere GPU and its NVIDIA driver (a Jetson Orin, sm_87, or a GeForce +RTX 30, sm_86). Needs no CUDA toolkit there, only Python 3 and the driver's libcuda: the driver compiles each op's PTX itself. + +For each op in ops.py: a PTX kernel that loads its inputs from global memory, does the op and stores its outputs. On the device (over ssh) the driver compiles +and runs it on the inputs made here, and prog.py rewrites the same machine code for the translator (LDG / STG become LDS / STS, the CUDA parameter reads and +the addressing they feed become NOPs), which the device's nvdisasm checks. The fixture keeps, per op, that program, its disassembly, the inputs and the GPU's +outputs. + + capture.py --host user@machine [--arch sm_87] [--cases 1024] [--only names,families] [--out dm/gfx/tests/ampere_orin] + +The device needs nvdisasm in --device-dir (NVIDIA's cuda_nvdisasm redistributable for linux-aarch64 or linux-x86_64; no install). PTX is written here, so no +compiler is needed on either side. Needs numpy here (.github/requirements/tools.txt).""" +import argparse +import gzip +import json +import re +import subprocess +import sys +from pathlib import Path + +import numpy as np # pyright: ignore[reportMissingImports] # numpy is in .github/requirements/tools.txt, not in the lint environment + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from ops import OPS # noqa: E402 + +FIXTURE = HERE.parents[1] / 'tests' / 'ampere_orin' + +F32_SPECIAL = [0x00000000, 0x80000000, 0x00000001, 0x807fffff, 0x00800000, 0x00000100, 0x3f800000, 0xbf800000, 0x3f000000, 0x7f7fffff, 0xff7fffff, + 0x7f800000, 0xff800000, 0x7fc00000, 0xffc00000, 0x7f800001, 0x7fffffff, 0x4b800000, 0x4b800001, 0x3effffff, 0x3f7fffff, 0x40490fdb, + 0x4f000000, 0xcf000000, 0x4f800000, 0x5f000000, 0x3fc00000, 0x40200000, 0xbfc00000, 0x33800000, 0x38800000, 0x477fe000, 0x477ff000, + 0x33000000, 0x3f800001, 0x4effffff, 0xceffffff, 0x3f7ffffe, 0x4b000000, 0xcb000000, 0x4b7fffff, 0x3ea2f983, 0x40c90fdb, 0x42b17218] +F16_SPECIAL = [0x0000, 0x8000, 0x0001, 0x03ff, 0x0400, 0x3c00, 0xbc00, 0x3800, 0x7bff, 0xfbff, 0x7c00, 0xfc00, 0x7e00, 0x7c01, 0x7fff, 0xfe00, + 0x3555, 0x3bff, 0x3c01, 0x0200, 0x8001, 0x6400, 0x4248, 0x5640] +BF16_SPECIAL = [0x0000, 0x8000, 0x0001, 0x007f, 0x0080, 0x3f80, 0xbf80, 0x3f00, 0x7f7f, 0xff7f, 0x7f80, 0xff80, 0x7fc0, 0x7f81, 0x7fff, 0x4049] +U32_SPECIAL = [0, 1, 2, 3, 7, 0xffffffff, 0xfffffffe, 0x80000000, 0x7fffffff, 0x7ffffffe, 0x80000001, 0xff, 0x100, 0xffff, 0x10000, 0x55555555, + 0xaaaaaaaa, 31, 32, 33, 63, 64, 0xffffffe1, 0x00ff00ff, 0x12345678, 0x80808080, 0x7f7f7f7f] + + +def _halves(kind, m, rng, tame): + def pack(v): + return (v.astype(np.float16).view(np.uint16) if kind == 'h' else (v.astype(np.float32).view(np.uint32) >> 16).astype(np.uint16)).astype(np.uint32) + if tame: + return pack(rng.uniform(-1, 1, m)) + p = rng.random(m) + a = rng.choice(np.array(F16_SPECIAL if kind == 'h' else BF16_SPECIAL, np.uint32), m) + b = rng.integers(0, 2 ** 16, m).astype(np.uint32) + c = pack(rng.uniform(-4, 4, m)) + d = pack(rng.choice([-1.0, 1.0], m) * np.exp2(rng.uniform(-14, 15, m))) + return np.select([p < .3, p < .45, p < .75], [a, b, c], d).astype(np.uint32) + + +def gen_kind(kind, n, rng): + """n input words of one kind (ops.py): special values, random bits and ordinary values mixed; an upper-case kind gives ordinary values only""" + tame, k = kind.isupper(), kind.lower() + pick = rng.random(n) + rand = rng.integers(0, 2 ** 32, n, dtype=np.uint64).astype(np.uint32) + if k == 'f': + if tame: + return rng.uniform(-2, 2, n).astype(np.float32).view(np.uint32) + a = rng.choice(np.array(F32_SPECIAL, np.uint32), n) + c = rng.uniform(-8, 8, n).astype(np.float32).view(np.uint32) + d = (rng.choice([-1.0, 1.0], n) * np.exp2(rng.uniform(-30, 30, n))).astype(np.float32).view(np.uint32) + e = (rng.integers(-4, 5, n) * 0.5).astype(np.float32).view(np.uint32) # halves and whole numbers: the ties of the roundings + return np.select([pick < .25, pick < .4, pick < .65, pick < .85], [a, rand, c, d], e).astype(np.uint32) + if k in 'hb': + return (_halves(k, n, rng, tame) | _halves(k, n, rng, tame) << 16).astype(np.uint32) + if k == 'u': + if tame: + return rand + a = rng.choice(np.array(U32_SPECIAL, np.uint32), n) + c = rng.integers(-1000, 1000, n).astype(np.int32).view(np.uint32) + return np.select([pick < .3, pick < .7], [a, rand], c).astype(np.uint32) + if k == 's': + a = rng.integers(0, 41, n).astype(np.uint32) + c = rng.choice(np.array([0, 1, 31, 32, 33, 63, 64, 255, 256, 0xffffffff, 0x80000000, 0xffffffe0], np.uint32), n) + return np.select([pick < .7, pick < .85], [a, rand], c).astype(np.uint32) + if k == 'l': + return np.where(pick < .8, rng.integers(0, 32, n).astype(np.uint32), rand).astype(np.uint32) + if k == 'p': # whole warps of one value now and then, so all / any / uni see both answers + same = np.repeat(rng.random(n // 32) < .3, 32) + return np.where(same, np.repeat(rng.integers(0, 2 ** 32, n // 32, dtype=np.uint64).astype(np.uint32), 32), rand).astype(np.uint32) + raise ValueError(kind) + + +PTX_HEAD = '''.version 8.5 +.target {arch} +.address_size 64 +.visible .entry k(.param .u64 pin, .param .u64 pout) +{ + .reg .b32 %i<{nin}>, %o<{nout}>, %t<8>, %r<8>, %x<8>; + .reg .b16 %y<8>, %w<8>; + .reg .f32 %f<8>; + .reg .b64 %d<8>, %a<4>; + .reg .f64 %g<4>; + .reg .pred %p<4>; + mov.u32 %r0, %tid.x; mov.u32 %r1, %ctaid.x; shl.b32 %r1, %r1, 5; add.u32 %r2, %r1, %r0; + ld.param.u64 %a0, [pin]; cvta.to.global.u64 %a0, %a0; mul.wide.u32 %a1, %r2, {inb}; add.u64 %a0, %a0, %a1; +''' + + +def ptx_for(o, arch): + """the op's kernel: volatile loads and stores, so the compiler keeps one 32-bit LDG / STG a word, which prog.py replaces one for one""" + nin, nout = len(o['ins']), len(o['outs']) + s = PTX_HEAD.replace('{arch}', arch).replace('{nin}', str(max(nin, 1))).replace('{nout}', str(nout)).replace('{inb}', str(nin * 4)) + s += ''.join(f' ld.volatile.global.b32 %i{k}, [%a0+{k * 4}];\n' for k in range(nin)) + s += ' ' + o['body'] + '\n' + s += f' ld.param.u64 %a2, [pout]; cvta.to.global.u64 %a2, %a2; mul.wide.u32 %a3, %r2, {nout * 4}; add.u64 %a2, %a2, %a3;\n' + s += ''.join(f' st.volatile.global.b32 [%a2+{k * 4}], %o{k};\n' for k in range(nout)) + return s + ' ret;\n}\n' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--host', required=True, help='ssh destination of the machine with the Ampere GPU') + ap.add_argument('--arch', default='sm_87', help='the GPU\'s PTX target: sm_87 (Jetson Orin), sm_86 (GeForce RTX 30)') + ap.add_argument('--cases', type=int, default=1024, help='cases an op, a multiple of 32 (one warp a group, as on the Metal side)') + ap.add_argument('--only', default='', help='op names or families, comma-separated') + ap.add_argument('--out', default=str(FIXTURE)) + ap.add_argument('--device-dir', default='ampere_orin', help='folder on the device (relative to its home) holding nvdisasm; the run uses /run') + ap.add_argument('--work', default='', help='local folder for the device\'s files (default: a temporary one)') + args = ap.parse_args() + if args.cases % 32: + sys.exit('--cases must be a multiple of 32') + want = set(args.only.split(',')) if args.only else None + ops = [o for o in OPS if not want or o['name'] in want or o['family'] in want] + import tempfile + with tempfile.TemporaryDirectory(prefix='ampere_orin_') as tmp: + work = Path(args.work or tmp) + work.mkdir(parents=True, exist_ok=True) + man = [] + for idx, o in enumerate(ops): + rng = np.random.default_rng(0x5eed + idx * 7919 + sum(map(ord, o['name']))) + data = np.stack([gen_kind(k, args.cases, rng) for k in o['ins']], 1) + (work / f"{o['name']}.ptx").write_text(ptx_for(o, args.arch)) + data.astype('/dev/null'], stdout=subprocess.PIPE) + subprocess.run(['tar', 'xf', '-', '-C', str(work)], stdin=tar.stdout, check=False) + tar.wait() + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + write_fixture(out, work, man, ops, args.arch) + + +def trimmed_sass(text): + """a program's disassembly without the hex column and the NOP padding after its last EXIT""" + lines = [ln for ln in text.splitlines() if ln.startswith('//') or '/*0' in ln] + lines = [re.sub(r'\s*/\* 0x[0-9a-f]+ \*/', '', ln).rstrip() for ln in lines] + while lines and lines[-1].rstrip(' ;').endswith('NOP'): + lines.pop() + out, run = [], [] + for ln in lines + ['']: + if ln.rstrip(' ;').endswith('NOP'): + run.append(ln) + continue + out += run if len(run) < 3 else [run[0], f' ... ({len(run) - 2} more NOPs)', run[-1]] + run = [] + out.append(ln) + return out[:-1] + + +def write_fixture(out, work, man, ops, arch): + """out/manifest.json (the device; per op its kinds, PTX body, case count and where its words lie), out/ampere_orin.u32.gz (per op: the program, then its + cases, a row of input words and the GPU's output words each) and out/ampere_orin.sass (each program's disassembly). Ops this run did not record keep their + old entries""" + info = json.loads((work / 'device.info').read_text()) if (work / 'device.info').exists() else {} + old_doc = json.loads((out / 'manifest.json').read_text()) if (out / 'manifest.json').exists() else {'ops': []} + old_blob = gzip.decompress((out / 'ampere_orin.u32.gz').read_bytes()) if (out / 'ampere_orin.u32.gz').exists() else b'' + old_sass = {} + if (out / 'ampere_orin.sass').exists(): + for part in (out / 'ampere_orin.sass').read_text().split('\n// == ')[1:]: + name, _, body = part.partition('\n') + old_sass[name.strip()] = body.rstrip('\n').splitlines() + entries = {} + for o in old_doc.get('ops', []): + if 'prog' not in o: + continue + prog = old_blob[o['prog'][0]:o['prog'][0] + o['prog'][1]] + rows = old_blob[o['rows'][0]:o['rows'][0] + o['rows'][1]] + entries[o['name']] = ({k: v for k, v in o.items() if k not in ('prog', 'rows')}, prog, rows, old_sass.get(o['name'], [])) + for o in man: + n = o['name'] + if not (work / f'{n}.orin.bin').exists() or not (work / f'{n}.prog.bin').exists(): + err = (work / f'{n}.err').read_text().splitlines()[0] if (work / f'{n}.err').exists() else 'no result' + print(f'{n}: left out ({err})', file=sys.stderr) + continue + ins = np.fromfile(work / f'{n}.in.bin', '> 16, kind)] + return [(word, kind)] + + +def denorm_input(row, kinds): + for v, k in zip(row, kinds): + k = k.lower() + if k == 'f' and is_denorm(v, 'f'): + return True + return False + + +def _f32(v): + return struct.unpack(' 0 else 0xff800000 + + +def rn_reference(name, row): + """the round-to-nearest result of a directed-rounding op on one case, or None. Doubles hold a sum or product of two floats exactly (and an FMA's well enough + for this label), so rounding the double once to float is the nearest float""" + mode = name.rsplit('_', 1)[-1] + if mode not in ('rz', 'rm', 'rp'): + return None + if name.startswith(('fadd_', 'fmul_', 'ffma_')): + a, b = _f32(row[0]), _f32(row[1]) + if name.startswith('fadd_'): + r = a + b + elif name.startswith('fmul_'): + r = a * b + else: + r = a * b + _f32(row[2]) + return _bits(r) if r == r else None + if name.startswith('i2f_s32_'): + return _bits(float(row[0] - (1 << 32) if row[0] >> 31 else row[0])) + return None + + +def classify(op, rows, orin, metal): + """(Counter of causes over differing lanes, number of lanes, the largest ULP distance among 'other' float lanes, up to four differing cases as dicts)""" + causes, n, worst, examples = Counter(), 0, 0, [] + kinds_in, kinds_out = op['ins'], op['outs'] + for r, (row, o_row, m_row) in enumerate(zip(rows, orin, metal)): + din = None + rn = None + row_causes = [] + for k, (ow, mw) in enumerate(zip(o_row, m_row)): + kind = kinds_out[k] + for (a, lk), (b, _) in zip(lanes(ow, kind), lanes(mw, kind)): + n += 1 + if a == b: + continue + if lk == 'u': + if din is None: + din = denorm_input(row, kinds_in) + cause = 'denorm-in' if din else 'other' + elif is_nan(a, lk) and is_nan(b, lk): + cause = 'nan-bits' + else: + if din is None: + din = denorm_input(row, kinds_in) + if din: + cause = 'denorm-in' + elif is_nan(a, lk) or is_nan(b, lk): + cause = 'nan-one' + elif lk == 'f' and (is_denorm(a, lk) or is_denorm(b, lk) or {a & 0x7fffffff, b & 0x7fffffff} == {0, 0x00800000}): + cause = 'denorm-out' + elif is_zero(a, lk) and is_zero(b, lk): + cause = 'zero-sign' + else: + if rn is None and len(kinds_out) == 1: + rn = rn_reference(op['name'], row) or -1 + if rn == b: + cause = 'rn-result' + else: + cause = 'other' + worst = max(worst, ulps(a, b, lk)) + causes[cause] += 1 + row_causes.append(cause) + if row_causes and len(examples) < 4 and any(c not in PLATFORM for c in row_causes): + examples.append({'lane': r % 32, 'in': [f'{v:#010x}' for v in row], 'gpu': [f'{v:#010x}' for v in o_row], + 'metal': [f'{v:#010x}' for v in m_row], 'why': sorted(set(row_causes))}) + return causes, n, worst, examples diff --git a/dm/gfx/tools/ampere_orin/cu.py b/dm/gfx/tools/ampere_orin/cu.py new file mode 100644 index 0000000000..beaa130572 --- /dev/null +++ b/dm/gfx/tools/ampere_orin/cu.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Runs a kernel on the Jetson through libcuda alone (no toolkit, no numpy): the driver JIT-compiles PTX for this GPU, or loads a cubin as given. + + cu.py ptx2cubin k.ptx k.cubin JIT the PTX for this GPU and save the cubin the linker produced + cu.py run k.(ptx|cubin) kernel grid block shared in.bin out_bytes out.bin [more in.bin ...] + launches kernel(in0, [in1, ...,] out) with grid x block threads and `shared` bytes of dynamic shared memory + +Every buffer is a plain device allocation; the kernel gets their addresses in argument order, the output last.""" +import ctypes as C +import sys + +cu = C.CDLL('libcuda.so.1') + + +def ck(r, what): + if r: + s = C.c_char_p() + cu.cuGetErrorString(r, C.byref(s)) + raise SystemExit(f'{what}: {r} {s.value.decode() if s.value else ""}') + + +def init(): + ck(cu.cuInit(0), 'cuInit') + dev = C.c_int() + ck(cu.cuDeviceGet(C.byref(dev), 0), 'cuDeviceGet') + ctx = C.c_void_p() + ck(cu.cuDevicePrimaryCtxRetain(C.byref(ctx), dev), 'cuDevicePrimaryCtxRetain') + ck(cu.cuCtxSetCurrent(ctx), 'cuCtxSetCurrent') + return dev + + +CU_JIT_INFO_LOG_BUFFER, CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES = 3, 4 +CU_JIT_ERROR_LOG_BUFFER, CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES = 5, 6 +CU_JIT_INPUT_PTX = 1 + + +def ptx2cubin(ptx: bytes) -> bytes: + log = C.create_string_buffer(1 << 16) + opts = (C.c_int * 2)(CU_JIT_ERROR_LOG_BUFFER, CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES) + vals = (C.c_void_p * 2)(C.cast(log, C.c_void_p), C.c_void_p(len(log))) + st = C.c_void_p() + ck(cu.cuLinkCreate_v2(2, opts, vals, C.byref(st)), 'cuLinkCreate') + data = C.create_string_buffer(ptx + b'\0') + r = cu.cuLinkAddData_v2(st, CU_JIT_INPUT_PTX, data, C.c_size_t(len(ptx) + 1), b'k.ptx', 0, None, None) + if r: + sys.stderr.write(log.value.decode(errors='replace')) + ck(r, 'cuLinkAddData') + out, size = C.c_void_p(), C.c_size_t() + ck(cu.cuLinkComplete(st, C.byref(out), C.byref(size)), 'cuLinkComplete') + cubin = C.string_at(out, size.value) + cu.cuLinkDestroy(st) + return cubin + + +def run(image: bytes, kernel: str, grid: int, block: int, shared: int, inputs, out_bytes: int) -> bytes: + mod = C.c_void_p() + img = C.create_string_buffer(image + (b'\0' if image[:4] != b'\x7fELF' else b'')) + ck(cu.cuModuleLoadData(C.byref(mod), img), 'cuModuleLoadData') + fn = C.c_void_p() + ck(cu.cuModuleGetFunction(C.byref(fn), mod, kernel.encode()), 'cuModuleGetFunction') + if shared > 48 * 1024: + ck(cu.cuFuncSetAttribute(fn, 8, shared), 'cuFuncSetAttribute(max dynamic shared)') # CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES + ptrs = [] + for data in inputs: + p = C.c_uint64() + ck(cu.cuMemAlloc_v2(C.byref(p), C.c_size_t(max(len(data), 4))), 'cuMemAlloc') + ck(cu.cuMemcpyHtoD_v2(p, data, C.c_size_t(len(data))), 'cuMemcpyHtoD') + ptrs.append(p) + o = C.c_uint64() + ck(cu.cuMemAlloc_v2(C.byref(o), C.c_size_t(out_bytes)), 'cuMemAlloc') + ck(cu.cuMemsetD8_v2(o, 0xcd, C.c_size_t(out_bytes)), 'cuMemsetD8') # a fill no result is likely to equal, so an unwritten word shows + ptrs.append(o) + args = (C.c_void_p * len(ptrs))(*[C.cast(C.byref(p), C.c_void_p) for p in ptrs]) + ck(cu.cuLaunchKernel(fn, grid, 1, 1, block, 1, 1, shared, None, args, None), 'cuLaunchKernel') + ck(cu.cuCtxSynchronize(), 'kernel') + buf = C.create_string_buffer(out_bytes) + ck(cu.cuMemcpyDtoH_v2(buf, o, C.c_size_t(out_bytes)), 'cuMemcpyDtoH') + for p in ptrs: + cu.cuMemFree_v2(p) + cu.cuModuleUnload(mod) + return buf.raw + + +if __name__ == '__main__': + cu.cuGetErrorString.argtypes = [C.c_int, C.POINTER(C.c_char_p)] + cu.cuLinkAddData_v2.argtypes = [C.c_void_p, C.c_int, C.c_void_p, C.c_size_t, C.c_char_p, C.c_uint, C.c_void_p, C.c_void_p] + cu.cuMemcpyHtoD_v2.argtypes = [C.c_uint64, C.c_char_p, C.c_size_t] + cu.cuMemcpyDtoH_v2.argtypes = [C.c_void_p, C.c_uint64, C.c_size_t] + cu.cuMemsetD8_v2.argtypes = [C.c_uint64, C.c_ubyte, C.c_size_t] + cu.cuMemFree_v2.argtypes = [C.c_uint64] + cu.cuLaunchKernel.argtypes = [C.c_void_p] + [C.c_uint] * 7 + [C.c_void_p, C.c_void_p, C.c_void_p] + init() + cmd, a = sys.argv[1], sys.argv[2:] + if cmd == 'ptx2cubin': + open(a[1], 'wb').write(ptx2cubin(open(a[0], 'rb').read())) + elif cmd == 'run': + src, kernel, grid, block, shared, in0, out_bytes, out = a[:8] + image = open(src, 'rb').read() + if src.endswith('.ptx'): + image = ptx2cubin(image) + ins = [open(p, 'rb').read() for p in [in0] + a[8:]] + open(out, 'wb').write(run(image, kernel, int(grid), int(block), int(shared), ins, int(out_bytes))) + else: + raise SystemExit(__doc__) diff --git a/dm/gfx/tools/ampere_orin/ops.py b/dm/gfx/tools/ampere_orin/ops.py new file mode 100644 index 0000000000..6ea50aace1 --- /dev/null +++ b/dm/gfx/tools/ampere_orin/ops.py @@ -0,0 +1,237 @@ +"""The ops the batch checks: each a PTX body over 32-bit inputs %i0.. that writes 32-bit outputs %o0... + +Input kinds (gen.py makes the values): f = f32, h = f16x2, b = bf16x2, u = 32-bit integer, s = shift or bit count, l = lane index, p = predicate-ish bit. +Output kinds (compare.py reads them): f, h, b, u. An op is (name, family, inputs, outputs, body).""" + +OPS = [] + + +def op(name, family, ins, outs, body): + OPS.append(dict(name=name, family=family, ins=ins, outs=outs, body=body)) + + +def fbin(o, extra=''): # f32 a op b + return f'mov.b32 %f0, %i0; mov.b32 %f1, %i1; {o} %f2, %f0, %f1; mov.b32 %o0, %f2;' + + +def fun(o): # f32 op a + return f'mov.b32 %f0, %i0; {o} %f2, %f0; mov.b32 %o0, %f2;' + + +def fter(o): + return f'mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; {o} %f2, %f0, %f1, %f3; mov.b32 %o0, %f2;' + + +def hbin(o): # f16x2 a op b + return f'mov.b32 %x0, %i0; mov.b32 %x1, %i1; {o} %x2, %x0, %x1; mov.b32 %o0, %x2;' + + +def hter(o): + return f'mov.b32 %x0, %i0; mov.b32 %x1, %i1; mov.b32 %x3, %i2; {o} %x2, %x0, %x1, %x3; mov.b32 %o0, %x2;' + + +# --- f32 arithmetic (FADD / FMUL / FFMA / FMNMX), every rounding mode, ftz and sat +for rnd in ('rn', 'rz', 'rm', 'rp'): + op(f'fadd_{rnd}', 'f32 arith', 'ff', 'f', fbin(f'add.{rnd}.f32')) + op(f'fmul_{rnd}', 'f32 arith', 'ff', 'f', fbin(f'mul.{rnd}.f32')) + op(f'ffma_{rnd}', 'f32 arith', 'fff', 'f', fter(f'fma.{rnd}.f32')) +op('fadd_ftz', 'f32 arith', 'ff', 'f', fbin('add.rn.ftz.f32')) +op('fadd_sat', 'f32 arith', 'ff', 'f', fbin('add.rn.sat.f32')) +op('fsub', 'f32 arith', 'ff', 'f', fbin('sub.rn.f32')) +op('fmul_ftz', 'f32 arith', 'ff', 'f', fbin('mul.rn.ftz.f32')) +op('fmul_sat', 'f32 arith', 'ff', 'f', fbin('mul.rn.sat.f32')) +op('ffma_ftz', 'f32 arith', 'fff', 'f', fter('fma.rn.ftz.f32')) +op('ffma_sat', 'f32 arith', 'fff', 'f', fter('fma.rn.sat.f32')) +op('ffma_ftz_sat', 'f32 arith', 'fff', 'f', fter('fma.rn.ftz.sat.f32')) +op('fadd_abs_neg', 'f32 arith', 'ff', 'f', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; abs.f32 %f2, %f0; neg.f32 %f3, %f1; add.rn.f32 %f4, %f2, %f3; mov.b32 %o0, %f4;') +op('fmul_by_imm', 'f32 arith', 'f', 'f', 'mov.b32 %f0, %i0; mul.rn.f32 %f2, %f0, 0f3EAAAAAB; mov.b32 %o0, %f2;') +op('fmin', 'f32 arith', 'ff', 'f', fbin('min.f32')) +op('fmax', 'f32 arith', 'ff', 'f', fbin('max.f32')) +op('fmin_ftz', 'f32 arith', 'ff', 'f', fbin('min.ftz.f32')) +op('fmax_nan', 'f32 arith', 'ff', 'f', fbin('max.NaN.f32')) +op('fmin_nan', 'f32 arith', 'ff', 'f', fbin('min.NaN.f32')) +op('fcopysign', 'f32 arith', 'ff', 'f', fbin('copysign.f32')) +op('fabs', 'f32 arith', 'f', 'f', fun('abs.f32')) +op('fneg', 'f32 arith', 'f', 'f', fun('neg.f32')) + +# --- f32 compares (FSETP / FSET), many predicates in one kernel +CMPS = ('eq', 'ne', 'lt', 'le', 'gt', 'ge', 'equ', 'neu', 'ltu', 'leu', 'gtu', 'geu', 'num', 'nan') +op('fsetp', 'f32 compare', 'ff', 'u' * len(CMPS), 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; ' + ' '.join( + f'setp.{c}.f32 %p0, %f0, %f1; selp.u32 %o{k}, 1, 0, %p0;' for k, c in enumerate(CMPS))) +op('fsetp_ftz', 'f32 compare', 'ff', 'uuuu', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; ' + ' '.join( + f'setp.{c}.ftz.f32 %p0, %f0, %f1; selp.u32 %o{k}, 1, 0, %p0;' for k, c in enumerate(('eq', 'lt', 'gtu', 'ne')))) +op('fset_bf', 'f32 compare', 'ff', 'ff', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; set.lt.f32.f32 %f2, %f0, %f1; set.neu.f32.f32 %f3, %f0, %f1; mov.b32 %o0, %f2; mov.b32 %o1, %f3;') +op('fset_u32', 'f32 compare', 'ff', 'uu', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; set.ge.u32.f32 %o0, %f0, %f1; set.nan.u32.f32 %o1, %f0, %f1;') +op('ftestp', 'f32 compare', 'f', 'uuuuu', 'mov.b32 %f0, %i0; ' + ' '.join( + f'testp.{t}.f32 %p0, %f0; selp.u32 %o{k}, 1, 0, %p0;' for k, t in enumerate(('finite', 'infinite', 'number', 'normal', 'subnormal')))) +op('fsel_cmp', 'f32 compare', 'fff', 'f', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; mov.b32 %f3, %i2; setp.gt.f32 %p0, %f0, 0f00000000; selp.f32 %f2, %f1, %f3, %p0; mov.b32 %o0, %f2;') + +# --- MUFU and the precise sequences built on it +for o in ('ex2.approx.f32', 'ex2.approx.ftz.f32', 'lg2.approx.f32', 'lg2.approx.ftz.f32', 'rcp.approx.f32', 'rcp.approx.ftz.f32', 'rsqrt.approx.f32', + 'rsqrt.approx.ftz.f32', 'sqrt.approx.f32', 'sqrt.approx.ftz.f32', 'sin.approx.f32', 'cos.approx.f32', 'tanh.approx.f32'): + op('mufu_' + o.replace('.approx', '').replace('.f32', '').replace('.', '_'), 'mufu', 'f', 'f', fun(o)) +op('mufu_ex2_f16x2', 'mufu', 'h', 'h', 'mov.b32 %x0, %i0; ex2.approx.f16x2 %x2, %x0; mov.b32 %o0, %x2;') +op('mufu_tanh_f16x2', 'mufu', 'h', 'h', 'mov.b32 %x0, %i0; tanh.approx.f16x2 %x2, %x0; mov.b32 %o0, %x2;') +op('fdiv_rn', 'f32 precise', 'ff', 'f', fbin('div.rn.f32')) +op('fdiv_full', 'f32 precise', 'ff', 'f', fbin('div.full.f32')) +op('fdiv_approx', 'f32 precise', 'ff', 'f', fbin('div.approx.f32')) +op('frcp_rn', 'f32 precise', 'f', 'f', fun('rcp.rn.f32')) +op('fsqrt_rn', 'f32 precise', 'f', 'f', fun('sqrt.rn.f32')) + +# --- conversions (F2I, I2F, F2F, FRND, F2FP, I2I) +for rnd in ('rni', 'rzi', 'rmi', 'rpi'): + op(f'f2i_s32_{rnd}', 'convert', 'f', 'u', f'mov.b32 %f0, %i0; cvt.{rnd}.s32.f32 %o0, %f0;') + op(f'frnd_{rnd}', 'convert', 'f', 'f', f'mov.b32 %f0, %i0; cvt.{rnd}.f32.f32 %f2, %f0; mov.b32 %o0, %f2;') +op('f2i_u32_rzi', 'convert', 'f', 'u', 'mov.b32 %f0, %i0; cvt.rzi.u32.f32 %o0, %f0;') +op('f2i_s32_ftz', 'convert', 'f', 'u', 'mov.b32 %f0, %i0; cvt.rzi.ftz.s32.f32 %o0, %f0;') +op('f2i_s16_sat', 'convert', 'f', 'u', 'mov.b32 %f0, %i0; cvt.rzi.sat.s16.f32 %w0, %f0; cvt.s32.s16 %o0, %w0;') +op('f2i_u8_sat', 'convert', 'f', 'u', 'mov.b32 %f0, %i0; cvt.rni.sat.u8.f32 %w0, %f0; cvt.u32.u8 %o0, %w0;') +op('f2i_s64', 'convert', 'f', 'uu', 'mov.b32 %f0, %i0; cvt.rzi.s64.f32 %d0, %f0; mov.b64 {%o0, %o1}, %d0;') +for rnd in ('rn', 'rz', 'rm', 'rp'): + op(f'i2f_s32_{rnd}', 'convert', 'u', 'f', f'cvt.{rnd}.f32.s32 %f2, %i0; mov.b32 %o0, %f2;') +op('i2f_u32_rn', 'convert', 'u', 'f', 'cvt.rn.f32.u32 %f2, %i0; mov.b32 %o0, %f2;') +op('i2f_s16', 'convert', 'u', 'f', 'cvt.u16.u32 %w0, %i0; cvt.rn.f32.s16 %f2, %w0; mov.b32 %o0, %f2;') +op('i2f_s64', 'convert', 'uu', 'f', 'mov.b64 %d0, {%i0, %i1}; cvt.rn.f32.s64 %f2, %d0; mov.b32 %o0, %f2;') +op('f2f_f16_rn', 'convert', 'f', 'h', 'mov.b32 %f0, %i0; cvt.rn.f16.f32 %y0, %f0; mov.b16 %w0, %y0; cvt.u32.u16 %o0, %w0;') +op('f2f_f16_rz', 'convert', 'f', 'h', 'mov.b32 %f0, %i0; cvt.rz.f16.f32 %y0, %f0; mov.b16 %w0, %y0; cvt.u32.u16 %o0, %w0;') +op('f2f_f32_from_f16', 'convert', 'h', 'ff', 'mov.b32 {%y0, %y1}, %i0; cvt.f32.f16 %f2, %y0; cvt.f32.f16 %f3, %y1; mov.b32 %o0, %f2; mov.b32 %o1, %f3;') +op('f2fp_pack_f16x2', 'convert', 'ff', 'h', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.f16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;') +op('f2fp_pack_f16x2_relu', 'convert', 'ff', 'h', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.relu.f16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;') +op('f2fp_pack_bf16x2', 'convert', 'ff', 'b', 'mov.b32 %f0, %i0; mov.b32 %f1, %i1; cvt.rn.bf16x2.f32 %x2, %f0, %f1; mov.b32 %o0, %x2;') +op('f2f_tf32', 'convert', 'f', 'u', 'mov.b32 %f0, %i0; cvt.rna.tf32.f32 %o0, %f0;') +op('i2i_s8_sat', 'convert', 'u', 'u', 'cvt.sat.s8.s32 %w0, %i0; cvt.s32.s8 %o0, %w0;') +op('i2i_u16_sat', 'convert', 'u', 'u', 'cvt.sat.u16.s32 %w0, %i0; cvt.u32.u16 %o0, %w0;') +op('sext_s8', 'convert', 'u', 'u', 'cvt.s32.s8 %o0, %i0;') + +# --- f16x2 / bf16x2 (HADD2 / HMUL2 / HFMA2 / HMNMX2 / HSETP2 / HSET2) +op('hadd2', 'f16x2', 'hh', 'h', hbin('add.rn.f16x2')) +op('hadd2_ftz', 'f16x2', 'hh', 'h', hbin('add.rn.ftz.f16x2')) +op('hadd2_sat', 'f16x2', 'hh', 'h', hbin('add.rn.sat.f16x2')) +op('hsub2', 'f16x2', 'hh', 'h', hbin('sub.rn.f16x2')) +op('hmul2', 'f16x2', 'hh', 'h', hbin('mul.rn.f16x2')) +op('hmul2_ftz', 'f16x2', 'hh', 'h', hbin('mul.rn.ftz.f16x2')) +op('hfma2', 'f16x2', 'hhh', 'h', hter('fma.rn.f16x2')) +op('hfma2_ftz', 'f16x2', 'hhh', 'h', hter('fma.rn.ftz.f16x2')) +op('hfma2_sat', 'f16x2', 'hhh', 'h', hter('fma.rn.sat.f16x2')) +op('hfma2_relu', 'f16x2', 'hhh', 'h', hter('fma.rn.relu.f16x2')) +op('hmin2', 'f16x2', 'hh', 'h', hbin('min.f16x2')) +op('hmax2', 'f16x2', 'hh', 'h', hbin('max.f16x2')) +op('hmin2_nan', 'f16x2', 'hh', 'h', hbin('min.NaN.f16x2')) +op('hneg2', 'f16x2', 'h', 'h', 'mov.b32 %x0, %i0; neg.f16x2 %x2, %x0; mov.b32 %o0, %x2;') +op('habs2', 'f16x2', 'h', 'h', 'mov.b32 %x0, %i0; abs.f16x2 %x2, %x0; mov.b32 %o0, %x2;') +op('hadd2_imm', 'f16x2', 'h', 'h', 'mov.b32 %x0, %i0; mov.b32 %x1, 0x3C00BC00; add.rn.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2;') +op('hadd_f16', 'f16x2', 'hh', 'hh', 'mov.b32 {%y0, %y1}, %i0; mov.b32 {%y2, %y3}, %i1; add.rn.f16 %y4, %y0, %y2; mul.rn.f16 %y5, %y1, %y3; mov.b16 %w0, %y4; cvt.u32.u16 %o0, %w0; mov.b16 %w1, %y5; cvt.u32.u16 %o1, %w1;') +op('hsetp2', 'f16x2', 'hh', 'uuuuuuuu', 'mov.b32 %x0, %i0; mov.b32 %x1, %i1; ' + ' '.join( + f'setp.{c}.f16x2 %p0|%p1, %x0, %x1; selp.u32 %o{2*k}, 1, 0, %p0; selp.u32 %o{2*k+1}, 1, 0, %p1;' for k, c in enumerate(('lt', 'eq', 'gtu', 'nan')))) +op('hset2', 'f16x2', 'hh', 'hu', 'mov.b32 %x0, %i0; mov.b32 %x1, %i1; set.lt.f16x2.f16x2 %x2, %x0, %x1; mov.b32 %o0, %x2; set.geu.u32.f16x2 %o1, %x0, %x1;') +op('bfma2', 'bf16x2', 'bbb', 'b', hter('fma.rn.bf16x2')) +op('bmin2', 'bf16x2', 'bb', 'b', hbin('min.bf16x2')) +op('bmax2_nan', 'bf16x2', 'bb', 'b', hbin('max.NaN.bf16x2')) + +# --- integer ALU +op('iadd', 'int', 'uu', 'u', 'add.u32 %o0, %i0, %i1;') +op('iadd3', 'int', 'uuu', 'u', 'add.u32 %t0, %i0, %i1; sub.u32 %o0, %t0, %i2;') +op('iadd_sat', 'int', 'uu', 'u', 'add.sat.s32 %o0, %i0, %i1;') +op('isub_sat', 'int', 'uu', 'u', 'sub.sat.s32 %o0, %i0, %i1;') +op('iadd64', 'int', 'uuuu', 'uu', 'mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; add.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;') +op('isub64', 'int', 'uuuu', 'uu', 'mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; sub.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;') +op('imul_lo', 'int', 'uu', 'u', 'mul.lo.u32 %o0, %i0, %i1;') +op('imul_hi_u32', 'int', 'uu', 'u', 'mul.hi.u32 %o0, %i0, %i1;') +op('imul_hi_s32', 'int', 'uu', 'u', 'mul.hi.s32 %o0, %i0, %i1;') +op('imul_wide_s32', 'int', 'uu', 'uu', 'mul.wide.s32 %d0, %i0, %i1; mov.b64 {%o0, %o1}, %d0;') +op('imul_wide_u32', 'int', 'uu', 'uu', 'mul.wide.u32 %d0, %i0, %i1; mov.b64 {%o0, %o1}, %d0;') +op('imad_lo', 'int', 'uuu', 'u', 'mad.lo.u32 %o0, %i0, %i1, %i2;') +op('imad_hi_s32', 'int', 'uuu', 'u', 'mad.hi.s32 %o0, %i0, %i1, %i2;') +op('imul64', 'int', 'uuuu', 'uu', 'mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; mul.lo.u64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;') +op('imul24', 'int', 'uu', 'u', 'mul24.lo.s32 %o0, %i0, %i1;') +op('imin_s32', 'int', 'uu', 'u', 'min.s32 %o0, %i0, %i1;') +op('imax_u32', 'int', 'uu', 'u', 'max.u32 %o0, %i0, %i1;') +op('imin_s64', 'int', 'uuuu', 'uu', 'mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; min.s64 %d2, %d0, %d1; mov.b64 {%o0, %o1}, %d2;') +op('iabs', 'int', 'u', 'u', 'abs.s32 %o0, %i0;') +op('ineg', 'int', 'u', 'u', 'neg.s32 %o0, %i0;') +op('isad', 'int', 'uuu', 'u', 'sad.u32 %o0, %i0, %i1, %i2;') +op('idiv_u32', 'int div', 'uu', 'u', 'div.u32 %o0, %i0, %i1;') +op('irem_u32', 'int div', 'uu', 'u', 'rem.u32 %o0, %i0, %i1;') +op('idiv_s32', 'int div', 'uu', 'u', 'div.s32 %o0, %i0, %i1;') +op('irem_s32', 'int div', 'uu', 'u', 'rem.s32 %o0, %i0, %i1;') +op('idiv_u32_by_7', 'int div', 'u', 'u', 'div.u32 %o0, %i0, 7;') +op('isetp', 'int compare', 'uu', 'uuuuuu', ' '.join( + f'setp.{c} %p0, %i0, %i1; selp.u32 %o{k}, 1, 0, %p0;' for k, c in enumerate(('lt.s32', 'lt.u32', 'ge.s32', 'hi.u32', 'eq.s32', 'ne.u32')))) +op('isetp_and', 'int compare', 'uuuu', 'uu', 'setp.lt.s32 %p1, %i2, %i3; setp.lt.and.s32 %p0, %i0, %i1, %p1; selp.u32 %o0, 1, 0, %p0; ' + 'setp.ge.or.u32 %p2, %i0, %i1, %p1; selp.u32 %o1, 1, 0, %p2;') +op('isetp64', 'int compare', 'uuuu', 'uu', 'mov.b64 %d0, {%i0, %i1}; mov.b64 %d1, {%i2, %i3}; setp.lt.s64 %p0, %d0, %d1; selp.u32 %o0, 1, 0, %p0; ' + 'setp.lt.u64 %p1, %d0, %d1; selp.u32 %o1, 1, 0, %p1;') +op('plop3', 'int compare', 'uuu', 'u', 'setp.ne.u32 %p0, %i0, 0; setp.ne.u32 %p1, %i1, 0; setp.ne.u32 %p2, %i2, 0; xor.pred %p3, %p0, %p1; or.pred %p3, %p3, %p2; ' + 'not.pred %p3, %p3; selp.u32 %o0, 1, 0, %p3;') + +# --- bits +op('shl', 'bits', 'us', 'u', 'shl.b32 %o0, %i0, %i1;') +op('shr_u32', 'bits', 'us', 'u', 'shr.u32 %o0, %i0, %i1;') +op('shr_s32', 'bits', 'us', 'u', 'shr.s32 %o0, %i0, %i1;') +op('shl_imm', 'bits', 'u', 'uu', 'shl.b32 %o0, %i0, 7; shr.s32 %o1, %i0, 13;') +op('shf_l_wrap', 'bits', 'uus', 'u', 'shf.l.wrap.b32 %o0, %i0, %i1, %i2;') +op('shf_l_clamp', 'bits', 'uus', 'u', 'shf.l.clamp.b32 %o0, %i0, %i1, %i2;') +op('shf_r_wrap', 'bits', 'uus', 'u', 'shf.r.wrap.b32 %o0, %i0, %i1, %i2;') +op('shf_r_clamp', 'bits', 'uus', 'u', 'shf.r.clamp.b32 %o0, %i0, %i1, %i2;') +op('shl64', 'bits', 'uus', 'uu', 'mov.b64 %d0, {%i0, %i1}; shl.b64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;') +op('shr64_u', 'bits', 'uus', 'uu', 'mov.b64 %d0, {%i0, %i1}; shr.u64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;') +op('shr64_s', 'bits', 'uus', 'uu', 'mov.b64 %d0, {%i0, %i1}; shr.s64 %d1, %d0, %i2; mov.b64 {%o0, %o1}, %d1;') +LUTS = (0x96, 0xe8, 0xca, 0x01, 0x80, 0x6a, 0xf8, 0x1e, 0xb4, 0x3c) +op('lop3', 'bits', 'uuu', 'u' * len(LUTS), ' '.join(f'lop3.b32 %o{k}, %i0, %i1, %i2, {hex(l)};' for k, l in enumerate(LUTS))) +op('logic', 'bits', 'uu', 'uuuu', 'and.b32 %o0, %i0, %i1; or.b32 %o1, %i0, %i1; xor.b32 %o2, %i0, %i1; not.b32 %t0, %i0; and.b32 %o3, %t0, %i1;') +PRMT = ('', '.f4e', '.b4e', '.rc8', '.ecl', '.ecr', '.rc16') +op('prmt', 'bits', 'uuu', 'u' * len(PRMT), ' '.join(f'prmt.b32{m} %o{k}, %i0, %i1, %i2;' for k, m in enumerate(PRMT))) +op('prmt_imm', 'bits', 'uu', 'uuu', 'prmt.b32 %o0, %i0, %i1, 0x3210; prmt.b32 %o1, %i0, %i1, 0x5140; prmt.b32 %o2, %i0, %i1, 0x8880;') +op('bfe_u32', 'bits', 'uss', 'u', 'bfe.u32 %o0, %i0, %i1, %i2;') +op('bfe_s32', 'bits', 'uss', 'u', 'bfe.s32 %o0, %i0, %i1, %i2;') +op('bfi', 'bits', 'uuss', 'u', 'bfi.b32 %o0, %i0, %i1, %i2, %i3;') +op('bmsk_clamp', 'bits', 'ss', 'u', 'bmsk.clamp.b32 %o0, %i0, %i1;') +op('bmsk_wrap', 'bits', 'ss', 'u', 'bmsk.wrap.b32 %o0, %i0, %i1;') +op('szext_s', 'bits', 'us', 'u', 'szext.clamp.s32 %o0, %i0, %i1;') +op('popc', 'bits', 'u', 'u', 'popc.b32 %o0, %i0;') +op('popc64', 'bits', 'uu', 'u', 'mov.b64 %d0, {%i0, %i1}; popc.b64 %o0, %d0;') +op('clz', 'bits', 'u', 'u', 'clz.b32 %o0, %i0;') +op('clz64', 'bits', 'uu', 'u', 'mov.b64 %d0, {%i0, %i1}; clz.b64 %o0, %d0;') +op('bfind_u32', 'bits', 'u', 'u', 'bfind.u32 %o0, %i0;') +op('bfind_s32', 'bits', 'u', 'u', 'bfind.s32 %o0, %i0;') +op('bfind_shiftamt', 'bits', 'u', 'u', 'bfind.shiftamt.u32 %o0, %i0;') +op('brev', 'bits', 'u', 'u', 'brev.b32 %o0, %i0;') +op('fns', 'bits', 'uus', 'u', 'and.b32 %t0, %i2, 31; fns.b32 %o0, %i0, %i1, %t0;') +op('dp4a_u32', 'bits', 'uuu', 'u', 'dp4a.u32.u32 %o0, %i0, %i1, %i2;') +op('dp4a_s32', 'bits', 'uuu', 'u', 'dp4a.s32.s32 %o0, %i0, %i1, %i2;') +op('dp4a_su', 'bits', 'uuu', 'u', 'dp4a.s32.u32 %o0, %i0, %i1, %i2;') +op('dp2a_lo', 'bits', 'uuu', 'u', 'dp2a.lo.s32.s32 %o0, %i0, %i1, %i2;') +op('dp2a_hi', 'bits', 'uuu', 'u', 'dp2a.hi.u32.u32 %o0, %i0, %i1, %i2;') + +# --- warp (SHFL / VOTE / MATCH / REDUX): 32 threads to a group on both sides, so a lane's neighbours are the same +op('shfl_idx', 'warp', 'ul', 'u', 'shfl.sync.idx.b32 %o0, %i0, %i1, 0x1f, 0xffffffff;') +op('shfl_up', 'warp', 'us', 'uu', 'and.b32 %t0, %i1, 31; shfl.sync.up.b32 %o0|%p0, %i0, %t0, 0, 0xffffffff; selp.u32 %o1, 1, 0, %p0;') +op('shfl_down', 'warp', 'us', 'uu', 'and.b32 %t0, %i1, 31; shfl.sync.down.b32 %o0|%p0, %i0, %t0, 0x1f, 0xffffffff; selp.u32 %o1, 1, 0, %p0;') +op('shfl_bfly', 'warp', 'ul', 'u', 'shfl.sync.bfly.b32 %o0, %i0, %i1, 0x1f, 0xffffffff;') +op('shfl_bfly_imm', 'warp', 'u', 'u', 'shfl.sync.bfly.b32 %o0, %i0, 1, 0x1f, 0xffffffff;') +op('shfl_idx_seg', 'warp', 'ul', 'u', 'shfl.sync.idx.b32 %o0, %i0, %i1, 0x1807, 0xffffffff;') +op('vote', 'warp', 'p', 'uuuu', 'and.b32 %t0, %i0, 1; setp.ne.u32 %p0, %t0, 0; vote.sync.ballot.b32 %o0, %p0, 0xffffffff; vote.sync.any.pred %p1, %p0, 0xffffffff; ' + 'selp.u32 %o1, 1, 0, %p1; vote.sync.all.pred %p2, %p0, 0xffffffff; selp.u32 %o2, 1, 0, %p2; vote.sync.uni.pred %p3, %p0, 0xffffffff; selp.u32 %o3, 1, 0, %p3;') +op('activemask', 'warp', 'p', 'u', 'activemask.b32 %o0;') +op('match_any', 'warp', 'p', 'u', 'and.b32 %t0, %i0, 3; match.any.sync.b32 %o0, %t0, 0xffffffff;') +op('redux_add', 'warp', 'u', 'u', 'redux.sync.add.u32 %o0, %i0, 0xffffffff;') +op('redux_min_s32', 'warp', 'u', 'u', 'redux.sync.min.s32 %o0, %i0, 0xffffffff;') +op('redux_or', 'warp', 'u', 'u', 'redux.sync.or.b32 %o0, %i0, 0xffffffff;') + +# --- tensor (HMMA / IMMA): each thread holds its fragments, as the PTX layouts give them +op('hmma_16816_f16', 'tensor', 'HHHHHHHH', 'hh', 'mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%o0, %o1}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%i6, %i7};') +op('hmma_16816_f32', 'tensor', 'HHHHHHFFFF', 'ffff', 'mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; ' + 'mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; ' + 'mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;') +op('hmma_1688_f16', 'tensor', 'HHHHH', 'hh', 'mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%o0, %o1}, {%i0, %i1}, {%i2}, {%i3, %i4};') +op('hmma_16816_bf16', 'tensor', 'BBBBBBFFFF', 'ffff', 'mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; ' + 'mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; ' + 'mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;') +op('hmma_1688_tf32', 'tensor', 'FFFFFFFFFF', 'ffff', 'mov.b32 %f0, %i6; mov.b32 %f1, %i7; mov.b32 %f2, %i8; mov.b32 %f3, %i9; ' + 'mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%f4, %f5, %f6, %f7}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%f0, %f1, %f2, %f3}; ' + 'mov.b32 %o0, %f4; mov.b32 %o1, %f5; mov.b32 %o2, %f6; mov.b32 %o3, %f7;') +op('imma_16832_s8', 'tensor', 'UUUUUUUUUU', 'uuuu', 'mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%o0, %o1, %o2, %o3}, {%i0, %i1, %i2, %i3}, {%i4, %i5}, {%i6, %i7, %i8, %i9};') + +# --- f64 (DADD / DMUL / DFMA): rare in shaders, here for the coverage list +op('dadd', 'f64', 'uuuu', 'uu', 'mov.b64 %g0, {%i0, %i1}; mov.b64 %g1, {%i2, %i3}; add.rn.f64 %g2, %g0, %g1; mov.b64 {%o0, %o1}, %g2;') +op('dmul', 'f64', 'uuuu', 'uu', 'mov.b64 %g0, {%i0, %i1}; mov.b64 %g1, {%i2, %i3}; mul.rn.f64 %g2, %g0, %g1; mov.b64 {%o0, %o1}, %g2;') diff --git a/dm/gfx/tools/ampere_orin/prog.py b/dm/gfx/tools/ampere_orin/prog.py new file mode 100644 index 0000000000..233325e55b --- /dev/null +++ b/dm/gfx/tools/ampere_orin/prog.py @@ -0,0 +1,163 @@ +"""The Metal side's program, made from the very SASS the Orin ran (stdlib only, so the Jetson can run it and disassemble the result). + +The op's kernel is k(in, out): it loads its NIN input words from in[gid * NIN + k] (LDG), computes, and stores NOUT words to out[gid * NOUT + k] (STG). +On the Metal side the translator gets the same instructions, where: + - two instructions go first: S2R Rb = SR_TID.X; IMAD Rb = Rb * W * 4 (W = NIN + NOUT words a thread, Rb a register the kernel does not use) + - each LDG Rd, [Ra + 4k] becomes LDS Rd, [Rb + 4k]; each STG [Ra + 4k], Rs becomes STS [Rb + 4 (NIN + k)], Rs (same guard and control bits) + - each instruction that reads the CUDA parameter bank c[0x0] (the stack pointer, the global window, the two pointers) becomes a NOP +so a thread's words lie at shared memory word tid * W: its inputs, then its outputs. Branches are relative, so the two inserted instructions move nothing.""" +import re +import struct + +CTRL = ~((1 << 41) - 1) & (2 ** 64 - 1) # the control bits (stall, yield, barriers, wait mask, reuse): bits 105..127 of the instruction, 41..63 of its high word +WAIT_ALL = 0x3f << 52 + + +def text_section(cubin, kernel='k'): + shoff, = struct.unpack_from(' instruction text, from nvdisasm -hex""" + out = {} + for m in re.finditer(r'/\*([0-9a-f]{4,})\*/\s+(.*?)\s*;\s*/\* 0x', sass): + out[int(m.group(1), 16)] = m.group(2) + return out + + +def rewrite(cubin, sass, nin, nout): + code = text_section(cubin) + text = sass_lines(sass) + insts = [struct.unpack_from('= 255: + raise ValueError('no free register') + w = nin + nout + out, notes, seen_in, seen_out = [], [], set(), set() + # The driver's JIT starts every kernel with a one-time convergence barrier set-up: ISETP P0 on UR2; @P0 BRA past it; BMOV B0..B15. Offline-compiled + # shaders (the games') have none, so it goes, up to the branch's target + skip = 0 + if text.get(0, '').startswith('ISETP.NE.U32.AND P0, PT, RZ, UR2') and text.get(16, '').startswith('@P0 BRA'): + rel = (insts[1][0] >> 32) & 0xffffffff # BRA's offset from the next instruction, in bytes + skip = (32 + rel) // 16 + notes.append(f'nop 0x0000..{skip * 16:#06x}: the JIT\'s barrier set-up') + dead = address_only(insts, text, skip) + for n in sorted(dead): + notes.append(f'nop {n * 16:#06x} (feeds only the removed addressing): {text.get(n * 16, "")}') + out.append((0x0000000000007919 | rb << 16, 0x000e280000002100)) # S2R Rb, SR_TID.X (write barrier 0) + out.append((0x0000000000007824 | rb << 16 | rb << 24 | (w * 4) << 32, 0x000fe200078e00ff | WAIT_ALL)) # IMAD Rb, Rb, W*4, RZ + for n, (lo, hi) in enumerate(insts): + t = text.get(n * 16, '') + opc = lo & 0xfff + guard = lo & 0xf000 + off = (lo >> 40) & 0xffffff + if n < skip: + out.append((0x7918, hi & CTRL)) + elif opc == 0x981: # LDG + if not re.match(r'(@!?U?P\w+ )?LDG\.E(\.STRONG\.\w+)? R\d+, \[R\d+\.64(\+0x[0-9a-f]+)?\]', t): + raise ValueError(f'unexpected load: {t}') + k = off // 4 + if off % 4 or not 0 <= k < nin: + raise ValueError(f'load offset {off:#x}: {t}') + seen_in.add(k) + rd = (lo >> 16) & 0xff + out.append((0x984 | guard | rd << 16 | rb << 24 | (k * 4) << 40, 0x0800 | (hi & CTRL))) + elif opc == 0x986: # STG + if not re.match(r'(@!?U?P\w+ )?STG\.E(\.STRONG\.\w+)? \[R\d+\.64(\+0x[0-9a-f]+)?\], R\d+', t): + raise ValueError(f'unexpected store: {t}') + k = off // 4 + if off % 4 or not 0 <= k < nout: + raise ValueError(f'store offset {off:#x}: {t}') + seen_out.add(k) + rs = (lo >> 32) & 0xff + out.append((0x388 | guard | rb << 24 | rs << 32 | ((nin + k) * 4) << 40, 0x0800 | (hi & CTRL))) + elif n in dead: + out.append((0x7918, hi & CTRL)) + else: + if re.search(r'c\[0x[1-9a-f]\]', t): + notes.append(f'constant bank read kept {n * 16:#06x}: {t}') + out.append((lo, hi)) + if seen_in != set(range(nin)) or seen_out != set(range(nout)): + raise ValueError(f'loads {sorted(seen_in)} of {nin}, stores {sorted(seen_out)} of {nout}') + return b''.join(struct.pack('` runs some, `AMPERE_ORIN_REPORT=` writes the results as JSON. To record again: `python3 dm/gfx/tools/ampere_orin/capture.py --host [--arch sm_86]`, with NVIDIA's `nvdisasm` (the `cuda_nvdisasm` redistributable) in `~/ampere_orin` on that machine. +- **Found on:** no game; a Jetson Orin's results. The HMMA comparison it extends is PR #1897's. + +### HFMA2.MMA, the half-pair FMA on the tensor pipe + +- **Symptom:** a Switch 2 shader that holds HFMA2.MMA fails to translate (`NotSupportedException: Ampere instruction 235` or `435`), so that shader isn't built. The GPU compiler emits it for an ordinary `fma.rn.f16x2` and, far more often, to load a 32-bit constant into a register (`HFMA2.MMA R5, -RZ, RZ, 0, 4.768e-07` is the integer 8), which it does in integer division, 64-bit integer arithmetic, `mul24`, F2I to 64 bits, TF32 conversion and the reciprocal's denormal scaling. +- **Cause:** the translator had HFMA2 (opcode 0x031 in its low 9 bits) but not its tensor-pipe twin 0x035. The two encode alike: in the Orin fixture, `HFMA2.MMA R7, R0, R7, R6` and `HFMA2 R7, R0, R7, R6` differ only in opcode bit 2, and .FTZ (80), .SAT (77) and .RELU (79) sit where NAK puts HFMA2's (`sm70_encode.rs` has no MMA form of its own). Its .RELU, shared with HFMA2 and HMUL2, also turned a NaN result into 0 (a plain `max` with zero), where PTX's `.relu` keeps it a NaN. +- **Fix:** `InstEmitAmpereAlu.cs` registers 0x035 to HFMA2's emitter, so its register (0x235), immediate (0x435), constant (0x635) and uniform-register forms decode as HFMA2's do and give the same MSL: two `fma` on the unpacked halves, packed back to `half2`. A program built with 0x235 read as 0x231 gives identical MSL, so a rewrite of that pattern covers both. .RELU now selects the NaN through `isnan` before the `max`. A chain change. +- **Check:** `test_dmsh_ampere_orin.py`: 17 ops failed to translate before. Now `hfma2`, `hfma2_sat`, `hfma2_relu`, `f2f_tf32`, `idiv_u32`, `idiv_u32_by_7`, `irem_s32`, `iadd64`, `isub64`, `imul64` and `imul24` match the Orin in every lane beyond NaN bits (`hfma2_relu` differed in 314 of 2,048 lanes with the old .RELU: NaN results written as 0). `mufu_rcp` runs and differs by 1 ULP in 92 of 1,024 lanes, as `mufu_rcp_ftz` does (Metal's reciprocal), so it moved to `APPROX`. `hfma2_ftz` differs in 86 lanes because f16 denormals are not flushed, as `hadd2_ftz` and `hmul2_ftz` do; `fdiv_rn`, `f2i_s64`, `dadd` and `dmul` now stop at FCHK, F2I.S64, DADD and DMUL, which are not implemented. The 0x635 and uniform forms are not in the fixture and were not run on a GPU. +- **Found on:** no game; a Jetson Orin's results ([Ampere instructions checked against an Ampere GPU](#ampere-instructions-checked-against-an-ampere-gpu)). + ### The NDVE movie network by the host - **Symptom:** a Switch 2 game's movies and the title behind them run below 60 fps on a Mac with a smaller GPU, and the GPU profile (`DM_NVN_GPUPROF`) shows almost all of the frame in a dozen compute dispatches of `nn::ndve`'s movie network: Ocarina of Time's title at about 24 fps and 38 ms of GPU a frame on an M1 Pro with 16 GPU cores, 24.8 ms on an M1 Max ([issue 1375](https://github.com/seedds/nxnative/issues/1375)), 18.9 ms of compute on an M4 with 8 ([HMMA chains keep their matrices in the SIMD-group](#hmma-chains-keep-their-matrices-in-the-simd-group)); 11-14 ms of the 16.7 on an M6, an M4 Pro or an M5 Max.