diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_bench.log b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_bench.log new file mode 100644 index 000000000..4afb39fb2 --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_bench.log @@ -0,0 +1,21 @@ +[mlx-rocm] bound HIP device 0: gfx1151 (AMD Radeon 8060S Graphics) cus=20 warp=32 lds=64KB +[Memory] cache limit: 2.15 GB +[Load] wall: 0.285 s MLX peak: 0.00 GB +[Memory] after load: active 0.00 GB, cache 0.00 GB, active+cache 0.00 GB, phase peak 0.00 GB +[long-prompt] target=512 tokens -> using 512 tokens (max_context=None, reserved 128 for generation) +[phase] warmup_start monotonic_ns=873860871948324 boottime_ns=873860871948374 +[mlx-rocm] gfx1151 native_wmma=1 cus=20 warp=32 low_cu=0 host_WARP_SIZE=32 +[hipBLASLt caps] device 0 (gfx1151): bf16=1 fp8_e4m3=0 fp8_e5m2=0 int8=1 +[Memory] after warmup (prefill + warmup tokens): active 18.01 GB, cache 0.12 GB, active+cache 18.13 GB, phase peak 18.17 GB +[long-prompt] target=512 tokens -> using 512 tokens (max_context=None, reserved 128 for generation) +[phase] measured_start monotonic_ns=873873042710977 boottime_ns=873873042710997 +[phase] decode_start monotonic_ns=873874739164814 boottime_ns=873874739164834 +[phase] measured_end monotonic_ns=873877675065738 boottime_ns=873877675065758 +[Memory] after measured pass: active 18.01 GB, cache 0.17 GB, active+cache 18.18 GB, phase peak 18.66 GB +[Profile Results] + Prompt tokens: 512 + Generated tokens: 128 + Prefill: 1694.32 ms (302.19 tok/s) + Decode: 2935.90 ms (43.60 tok/s) + Decode path: engine + MLX peak memory: 18.66 GB diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_decode_kernels.csv b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_decode_kernels.csv new file mode 100644 index 000000000..a49758d54 --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_decode_kernels.csv @@ -0,0 +1,52 @@ +kernel,class,roles,port_units,calls,calls_per_token,total_ns,avg_ns,share_pct +"void mlx::core::rocm::qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int, int, bool)",qmv,,,14859,116.09,706264203,47531,39.613 +"void mlx::core::rocm::gather_qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, unsigned int const*, unsigned int const*, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, hip_bfloat16*, int, int, int, int, int, bool, bool, long)",gather_qmv,moe_expert_gemv,2065,5842,45.64,467734830,80064,26.234 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",binary,sampler_tail;ssm_step,2064;2067,11811,92.27,75280902,6374,4.222 +"void mlx::core::rocm::binary_g(float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",binary,ssm_step,2067,14605,114.1,61815449,4232,3.467 +"void mlx::core::rocm::gemv_batched_inline(float const*, float const*, float*, int, int, mlx::core::rocm::GemvBatchParams)",gemm,ssm_step,2067,5842,45.64,39780619,6809,2.231 +"void mlx::core::rocm::block_sort_kernel(float const*, unsigned int*, int, long, long, long, long)",sort,,,2921,22.82,38292952,13110,2.148 +"void mlx::core::rocm::copy_v(hip_bfloat16 const*, float*, unsigned int)",copy,sampler_tail;ssm_gated_norm;ssm_step,2064;2067,26416,206.38,35912853,1360,2.014 +"void mlx::core::rocm::rms_norm_kernel(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, float, unsigned int, long)",rms_norm,add_rms_join;add_rms_join_post_attn,2063,6731,52.59,29870307,4438,1.675 +"void mlx::core::rocm::gemv_single(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int)",gemm,,,2921,22.82,28067669,9609,1.574 +"void mlx::core::rocm::binary_vv(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",binary,add_rms_join;add_rms_join_post_attn;ssm_step,2063;2067,12446,97.23,23934143,1923,1.342 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",binary,ssm_gated_norm;ssm_step,2067,11684,91.28,20571906,1761,1.154 +"void mlx::core::rocm::copy_s(float const*, float*, unsigned int)",copy,ssm_step,2067,11684,91.28,16388515,1403,0.919 +"void mlx::core::rocm::copy_g_byval(float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",copy,ssm_step,2067,8763,68.46,15696777,1791,0.880 +Cijk_Alik_Bjlk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8,gemm,ssm_step,2067,2921,22.82,14268641,4885,0.800 +"void mlx::core::rocm::unary_v(float const*, float*, unsigned int)",unary,ssm_step,2067,8763,68.46,14070768,1606,0.789 +"void mlx::core::rocm::ternary_g(bool const*, float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",ternary,ssm_step,2067,5842,45.64,13905548,2380,0.780 +"void mlx::core::rocm::arange_kernel(int*, int, int, unsigned long)",copy,ssm_step,2067,11684,91.28,13423189,1149,0.753 +"void mlx::core::rocm::copy_v(float const*, hip_bfloat16*, unsigned int)",copy,moe_activation;sampler_tail;ssm_gated_norm;ssm_step,2064;2065;2067,8890,69.45,13265591,1492,0.744 +"void mlx::core::rocm::CV2IAsTypeBDV2IBroadcastACEV2IBroadcastCAFV2IMaximumDEGV2OSquareF_VC_V2_2297668033614959926_contiguous(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",compiled,moe_activation,2065,5842,45.64,12821809,2195,0.719 +"void mlx::core::rocm::copy_gg_byval(hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",copy,ssm_conv,,7390,57.73,12506124,1692,0.701 +"void mlx::core::rocm::BV2ISigmoidACV2IBroadcastABDV2IBroadcastBAEV2OMultiplyCD_V_V2_6142509188972423790_contiguous(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",compiled,ssm_silu,,2921,22.82,10142188,3472,0.569 +"void mlx::core::rocm::kernel_sdpav_1pass(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, hip_bfloat16 const*, mlx::core::rocm::AttnParams)",sdpa,,,762,5.95,9153734,12013,0.513 +"void mlx::core::rocm::col_reduce_small(hip_bfloat16 const*, hip_bfloat16*, mlx::core::rocm::ColReduceArgs, unsigned long)",reduce,,,2921,22.82,8387734,2872,0.470 +"void mlx::core::rocm::binary_ss(int const*, int const*, bool*, unsigned int)",binary,ssm_step,2067,5842,45.64,7838668,1342,0.440 +Cijk_Ailk_Bljk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8,gemm,ssm_step,2067,2921,22.82,7639224,2615,0.428 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,2921,22.82,7027372,2406,0.394 +"void mlx::core::(anonymous namespace)::depthwise_conv1d_kernel(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, mlx::core::ConvParams<1>)",conv,ssm_conv,,2921,22.82,6733185,2305,0.378 +"void mlx::core::rocm::binary_vs(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,2921,22.82,6361108,2178,0.357 +"void mlx::core::rocm::rms_norm_kernel(float const*, float const*, float*, float, unsigned int, long)",rms_norm,ssm_gated_norm,,2921,22.82,5926723,2029,0.332 +"void mlx::core::rocm::Bf4ISigmoidACf4IBroadcastABDf4IBroadcastBAEf4OMultiplyCD_V_f4_6142509188972423790_contiguous(float const*, float*, unsigned int)",compiled,ssm_silu,,2921,22.82,5595526,1916,0.314 +"void mlx::core::rocm::unary_v(float const*, float*, unsigned int)",unary,ssm_step,2067,2921,22.82,5592707,1915,0.314 +"void mlx::core::rocm::contiguous_scan(float const*, float*, int)",scan,ssm_step,2067,2921,22.82,5399681,1849,0.303 +"void mlx::core::rocm::arange_kernel(unsigned int*, unsigned int, unsigned int, unsigned long)",copy,moe_gather_indices,2065,5842,45.64,5211505,892,0.292 +"void mlx::core::rocm::strided_scan(float const*, float*, int, long, long)",scan,ssm_step,2067,2921,22.82,5119157,1753,0.287 +"void mlx::core::rocm::gather_axis_kernel(float const*, int const*, float*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long)",gather_scatter,,,2921,22.82,5114065,1751,0.287 +"void mlx::core::rocm::binary_g(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",binary,,,2921,22.82,5085612,1741,0.285 +"void mlx::core::rocm::row_reduce_simple_kernel(float const*, float*, unsigned long, int)",reduce,,,2921,22.82,5084396,1741,0.285 +"void mlx::core::rocm::binary_vs(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,2921,22.82,4927030,1687,0.276 +"void mlx::core::rocm::Cf4IAsTypeADf4OSigmoidCEf4IBroadcastDBFf4IBroadcastBDGf4IAddEFHf4ONegativeG_VV_V2f4_6142509188972423790_strided<1, unsigned int, 8>(hip_bfloat16 const*, hip::std::array, float const*, hip::std::array, float*, float*, hip::std::array, unsigned int)",compiled,,,2921,22.82,4896259,1676,0.275 +"void mlx::core::rocm::Ef4IBroadcastBCFf4IBroadcastCBGf4IAddEFHf4IBroadcastAGIf4IBroadcastGAJf4IDivideHIKf4IBroadcastJDLf4IBroadcastDJMf4OMultiplyKL_VVCS_f4f4f4_9971424201298772266_strided<1, unsigned int, 1>(float const*, hip::std::array, float const*, hip::std::array, float const*, float*, hip::std::array, unsigned int)",compiled,,,2921,22.82,4077494,1396,0.229 +"void mlx::core::rocm::arg_reduce_partial, 256>(hip_bfloat16 const*, hip_bfloat16*, unsigned int*, unsigned long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, long, int, int)",arg_reduce,sampler_tail,2064,127,0.99,1228980,9677,0.069 +"void mlx::core::rocm::copy_v(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",copy,sampler_tail,2064,127,0.99,429689,3383,0.024 +"void mlx::core::rocm::gather_rows_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, unsigned int, int)",gather_scatter,,,254,1.98,423680,1668,0.024 +"void mlx::core::rocm::gather_axis_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long)",gather_scatter,sampler_tail,2064,127,0.99,365236,2876,0.020 +"void mlx::core::rocm::scatter_axis_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long, long)",gather_scatter,sampler_tail,2064,127,0.99,304541,2398,0.017 +"void mlx::core::rocm::arg_reduce_final, 256>(hip_bfloat16 const*, unsigned int const*, unsigned int*, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, unsigned long, int)",arg_reduce,sampler_tail,2064,127,0.99,266088,2095,0.015 +"void mlx::core::rocm::gather_rows_kernel(unsigned int const*, int const*, unsigned int*, long, unsigned int, int)",gather_scatter,,,127,0.99,247700,1950,0.014 +"void mlx::core::rocm::affine_dequantize_packed_kernel(unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned long, int)",dequantize,,,127,0.99,237600,1871,0.013 +"void mlx::core::rocm::copy_v(unsigned int const*, int*, unsigned int)",copy,sampler_tail,2064,126,0.98,201576,1600,0.011 +"void mlx::core::rocm::copy_s(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",copy,,,12,0.09,16191,1349,0.001 +"void mlx::core::rocm::copy_g_byval(hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",copy,ssm_step,2067,2,0.02,3606,1803,0.000 diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_kernel_stats.csv b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_kernel_stats.csv new file mode 100644 index 000000000..01921d3b5 --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_kernel_stats.csv @@ -0,0 +1,66 @@ +"Name","Calls","TotalDurationNs","AverageNs","Percentage","MinNs","MaxNs","StdDev" +"void mlx::core::rocm::gather_qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, unsigned int const*, unsigned int const*, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, hip_bfloat16*, int, int, int, int, int, bool, bool, long)",6808,3710654166,545043.208872,63.23,75141,293577688,5103817.547445 +"void mlx::core::rocm::qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int, int, bool)",17084,814816279,47694.701417,13.88,3486,1360460,78489.913788 +"Cijk_Ailk_Bljk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8",3404,135273334,39739.522327,2.31,1723,20208517,438683.514676 +"void mlx::core::rocm::binary_g(float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",17020,86853627,5103.033314,1.48,1001,261451,10303.225309 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",13580,86601604,6377.143152,1.48,801,93816,8359.245894 +"void mlx::core::rocm::ternary_g(bool const*, float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",6808,79273755,11644.206081,1.35,1442,15337474,197285.643334 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",13524,72157780,5335.535345,1.23,841,1049487,57304.393237 +"void mlx::core::rocm::copy_g_byval(float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",10166,63680814,6264.097383,1.09,1122,527830,46768.098512 +"void mlx::core::rocm::affine_dequantize_packed_kernel(unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned long, int)",380,56544036,148800.094737,0.9635,1723,1156208,189289.940297 +"void mlx::core::rocm::gemv_batched_inline(float const*, float const*, float*, int, int, mlx::core::rocm::GemvBatchParams)",6716,45689105,6803.023377,0.7786,1362,30697,5103.387032 +"void mlx::core::rocm::block_sort_kernel(float const*, unsigned int*, int, long, long, long, long)",3404,44908982,13193.002938,0.7653,10820,68568,1936.079597 +"void mlx::core::rocm::copy_v(hip_bfloat16 const*, float*, unsigned int)",30554,44393802,1452.962034,0.7565,681,61595,2302.035664 +"Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT64x128x64_MI16x16x1_SN_LDSB0_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR0_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT4_2_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR0_PKA0_SGROB0_SIA1_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSM1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS32_WG16_8_1_WGMXCC1",104,44384651,426775.490385,0.7563,260648,1314665,252311.583225 +"void mlx::core::rocm::strided_scan(float const*, float*, int, long, long)",3404,44155583,12971.675382,0.7524,1242,1014422,96010.966740 +"void mlx::core::rocm::unary_v(float const*, float*, unsigned int)",10166,43997433,4327.900157,0.7497,842,619401,40188.880783 +"void mlx::core::rocm::rms_norm_kernel(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, float, unsigned int, long)",7844,38009816,4845.718511,0.6477,3927,137777,5213.046960 +"void mlx::core::rocm::binary_vv(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",14458,33407298,2310.644487,0.5693,1122,142266,3971.001119 +"Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT96x128x32_MI16x16x1_SN_LDSB0_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS1_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT3_4_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR1_PLR0_PKA0_SGROB0_SIA3_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB4_WSGRA0_WSGRB0_WS32_WG32_4_1_WGMXCC1",46,33105701,719689.152174,0.5641,678913,1374858,99637.826115 +"void mlx::core::rocm::gemv_single(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int)",3358,32311040,9622.108398,0.5506,8576,27411,787.087264 +"Cijk_Alik_Bjlk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8",3404,25982242,7632.856052,0.4428,4328,254958,23475.176336 +"Cijk_Alik_Bljk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8",46,22329099,485415.195652,0.3805,196328,12524922,1818035.864119 +"void mlx::core::rocm::kernel_sdpa_flash_wmma(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, float*, mlx::core::rocm::FAWmmaParams)",12,19727506,1643958.833333,0.3362,1598357,1675141,21039.435400 +"void mlx::core::rocm::copy_s(float const*, float*, unsigned int)",13616,19241881,1413.181625,0.3279,841,10700,1036.362477 +"void mlx::core::rocm::copy_v(float const*, hip_bfloat16*, unsigned int)",10360,19039798,1837.818340,0.3244,841,119463,4961.713852 +"void mlx::core::rocm::copy_gg_byval(hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",8608,18917012,2197.608271,0.3224,961,378630,7256.259237 +"void mlx::core::rocm::CV2IAsTypeBDV2IBroadcastACEV2IBroadcastCAFV2IMaximumDEGV2OSquareF_VC_V2_2297668033614959926_contiguous(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",6808,18565590,2727.025558,0.3164,1763,59992,4798.879040 +"Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT64x64x32_MI16x16x1_SN_LDSB0_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR0_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT2_2_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA0_SGROB0_SIA3_SS1_SPO0_SRVW0_SSO0_SVW2_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA2_VWB2_WSGRA0_WSGRB0_WS32_WG32_4_1_WGMXCC1",58,17278873,297911.603448,0.2944,254838,755937,97786.470837 +"void mlx::core::rocm::binary_g(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",3404,16742269,4918.410400,0.2853,1522,248706,27077.360363 +"void mlx::core::rocm::arange_kernel(int*, int, int, unsigned long)",13616,15801042,1160.476058,0.2693,601,10459,1369.606535 +"void mlx::core::(anonymous namespace)::depthwise_conv1d_kernel(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, mlx::core::ConvParams<1>)",3404,14650431,4303.886898,0.2497,1723,664165,19189.525116 +"void mlx::core::rocm::BV2ISigmoidACV2IBroadcastABDV2IBroadcastBAEV2OMultiplyCD_V_V2_6142509188972423790_contiguous(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",3404,13228932,3886.290247,0.2254,2765,156453,4200.977416 +"void mlx::core::rocm::col_reduce_small(hip_bfloat16 const*, hip_bfloat16*, mlx::core::rocm::ColReduceArgs, unsigned long)",3404,11097895,3260.251175,0.1891,2444,111369,3548.726040 +"void mlx::core::rocm::kernel_sdpav_1pass(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, hip_bfloat16 const*, mlx::core::rocm::AttnParams)",876,10429047,11905.304795,0.1777,9217,44844,1297.683979 +"void mlx::core::rocm::binary_ss(int const*, int const*, bool*, unsigned int)",6716,9015878,1342.447588,0.1536,881,10620,925.232226 +"void mlx::core::rocm::rms_norm_kernel(float const*, float const*, float*, float, unsigned int, long)",3404,8645564,2539.824912,0.1473,1562,45165,4387.307642 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",3404,8283763,2433.537897,0.1412,1763,14547,1220.681224 +"void mlx::core::rocm::Bf4ISigmoidACf4IBroadcastABDf4IBroadcastBAEf4OMultiplyCD_V_f4_6142509188972423790_contiguous(float const*, float*, unsigned int)",3404,7615546,2237.234430,0.1298,1322,33303,2766.183925 +"void mlx::core::rocm::binary_vs(float const*, float const*, float*, unsigned int)",3404,7464941,2192.990893,0.1272,1283,16952,1665.911242 +"void mlx::core::rocm::copy_g_byval(hip_bfloat16 const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",230,7125749,30981.517391,0.1214,1843,210194,27672.063383 +"void mlx::core::rocm::unary_v(float const*, float*, unsigned int)",3404,6562489,1927.875734,0.1118,1162,11342,1584.760561 +"void mlx::core::rocm::contiguous_scan(float const*, float*, int)",3358,6281692,1870.664681,0.1070,1202,12824,1346.468377 +"void mlx::core::rocm::arange_kernel(unsigned int*, unsigned int, unsigned int, unsigned long)",6808,6125631,899.769536,0.1044,601,8896,247.292907 +"void mlx::core::rocm::row_reduce_simple_kernel(float const*, float*, unsigned long, int)",3404,5991588,1760.160987,0.1021,1242,10899,405.458478 +"void mlx::core::rocm::gather_axis_kernel(float const*, int const*, float*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long)",3358,5927145,1765.081894,0.1010,1242,7895,340.365129 +"void mlx::core::rocm::binary_vs(float const*, float const*, float*, unsigned int)",3404,5796693,1702.906287,0.0988,962,10900,1422.186444 +"void mlx::core::rocm::Cf4IAsTypeADf4OSigmoidCEf4IBroadcastDBFf4IBroadcastBDGf4IAddEFHf4ONegativeG_VV_V2f4_6142509188972423790_strided<1, unsigned int, 8>(hip_bfloat16 const*, hip::std::array, float const*, hip::std::array, float*, float*, hip::std::array, unsigned int)",3358,5651357,1682.953246,0.0963,1202,7334,290.643531 +"void mlx::core::rocm::binary_g(float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",92,5197386,56493.326087,0.0886,1803,115136,54137.563204 +"void mlx::core::rocm::Ef4IBroadcastBCFf4IBroadcastCBGf4IAddEFHf4IBroadcastAGIf4IBroadcastGAJf4IDivideHIKf4IBroadcastJDLf4IBroadcastDJMf4OMultiplyKL_VVCS_f4f4f4_9971424201298772266_strided<1, unsigned int, 1>(float const*, hip::std::array, float const*, hip::std::array, float const*, float*, hip::std::array, unsigned int)",3358,4705231,1401.200417,0.0802,921,5290,251.368410 +"void mlx::core::rocm::binary_g(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",46,3954696,85971.652174,0.0674,83316,142147,8690.816262 +"Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT64x64x64_MI16x16x1_SN_LDSB0_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR0_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT2_2_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA0_SGROB0_SIA1_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSM1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS32_WG32_4_1_WGMXCC1",70,2822133,40316.185714,0.0481,21720,112652,33459.513238 +"void mlx::core::rocm::copy_g_byval(hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",58,2394090,41277.413793,0.0408,1283,315271,95842.487996 +"void mlx::core::rocm::arg_reduce_partial, 256>(hip_bfloat16 const*, hip_bfloat16*, unsigned int*, unsigned long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, long, int, int)",148,1436890,9708.716216,0.0245,8697,17834,1641.001267 +"void mlx::core::rocm::binary_g(int const*, int const*, bool*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",92,775899,8433.684783,0.0132,7814,13506,1132.964253 +"void mlx::core::rocm::gather_rows_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, unsigned int, int)",296,533121,1801.084459,9.085e-03,1282,25367,1559.924667 +"void mlx::core::rocm::copy_v(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",148,494332,3340.081081,8.424e-03,2524,7935,1447.218945 +"void mlx::core::rocm::gather_axis_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long)",148,427553,2888.871622,7.286e-03,2004,7694,1624.578008 +"void mlx::core::rocm::scatter_axis_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long, long)",148,343417,2320.385135,5.852e-03,1683,10139,1506.199224 +"void mlx::core::rocm::gather_rows_kernel(unsigned int const*, int const*, unsigned int*, long, unsigned int, int)",148,312624,2112.324324,5.327e-03,1643,24245,1883.514374 +"void mlx::core::rocm::arg_reduce_final, 256>(hip_bfloat16 const*, unsigned int const*, unsigned int*, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, unsigned long, int)",148,308730,2086.013514,5.261e-03,1843,6893,561.184777 +"void mlx::core::rocm::copy_v(unsigned int const*, int*, unsigned int)",144,230432,1600.222222,3.927e-03,1363,2244,164.811545 +"void mlx::core::rocm::Cf4IAsTypeADf4OSigmoidCEf4IBroadcastDBFf4IBroadcastBDGf4IAddEFHf4ONegativeG_VV_V2f4_6142509188972423790_strided<2, unsigned int, 8>(hip_bfloat16 const*, hip::std::array, float const*, hip::std::array, float*, float*, hip::std::array, unsigned int)",46,208910,4541.521739,3.560e-03,3326,15389,2546.526669 +"void mlx::core::rocm::gather_axis_kernel(float const*, int const*, float*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long)",46,157053,3414.195652,2.676e-03,2084,11863,2294.093068 +"void mlx::core::rocm::copy_s(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",70,130445,1863.500000,2.223e-03,1122,10059,1493.859351 +"void mlx::core::rocm::Ef4IBroadcastBCFf4IBroadcastCBGf4IAddEFHf4IBroadcastAGIf4IBroadcastGAJf4IDivideHIKf4IBroadcastJDLf4IBroadcastDJMf4OMultiplyKL_VVCS_f4f4f4_9971424201298772266_strided<2, unsigned int, 1>(float const*, hip::std::array, float const*, hip::std::array, float const*, float*, hip::std::array, unsigned int)",46,121749,2646.717391,2.075e-03,1242,9738,1974.582923 +"__amd_rocclr_fillBufferUnAligned",1,75181,75181.000000,1.281e-03,75181,75181,0.00000000e+00 diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_plain_bench.log b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_plain_bench.log new file mode 100644 index 000000000..fd7aa85d1 --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_plain_bench.log @@ -0,0 +1,17 @@ +[mlx-rocm] bound HIP device 0: gfx1151 (AMD Radeon 8060S Graphics) cus=20 warp=32 lds=64KB +[Memory] cache limit: 2.15 GB +[Load] wall: 0.335 s MLX peak: 0.00 GB +[Memory] after load: active 0.00 GB, cache 0.00 GB, active+cache 0.00 GB, phase peak 0.00 GB +[long-prompt] target=512 tokens -> using 512 tokens (max_context=None, reserved 128 for generation) +[mlx-rocm] gfx1151 native_wmma=1 cus=20 warp=32 low_cu=0 host_WARP_SIZE=32 +[hipBLASLt caps] device 0 (gfx1151): bf16=1 fp8_e4m3=0 fp8_e5m2=0 int8=1 +[Memory] after warmup (prefill + warmup tokens): active 18.01 GB, cache 0.08 GB, active+cache 18.09 GB, phase peak 18.17 GB +[long-prompt] target=512 tokens -> using 512 tokens (max_context=None, reserved 128 for generation) +[Memory] after measured pass: active 18.01 GB, cache 0.17 GB, active+cache 18.18 GB, phase peak 18.66 GB +[Profile Results] + Prompt tokens: 512 + Generated tokens: 128 + Prefill: 1715.69 ms (298.42 tok/s) + Decode: 2473.41 ms (51.75 tok/s) + Decode path: engine + MLX peak memory: 18.66 GB diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_summary.json b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_summary.json new file mode 100644 index 000000000..9980eef73 --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_summary.json @@ -0,0 +1,263 @@ +{ + "name": "NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy", + "model_type": "nemotron_h", + "temperature": 0.0, + "clock": "boottime", + "prompt_tokens": 512, + "generated_tokens": 128, + "profiled_decode_tok_s": 43.6, + "profiled_prefill_tok_s": 302.19, + "plain_decode_tok_s": 51.75, + "plain_prefill_tok_s": 298.42, + "profiler_decode_slowdown_pct": 18.69, + "decode_dispatches": 251243, + "dispatches_per_token": 1962.8, + "decode_wall_ms": 2935.901, + "decode_gpu_sum_ms": 1782.911, + "decode_gpu_busy_ms": 1782.911, + "gpu_ms_per_token": 13.929, + "host_gap_ms_per_token": 9.0077, + "host_gap_pct_of_wall": 39.27, + "plain_wall_ms_per_token": 19.3237, + "plain_host_gap_ms_per_token_est": 5.3947, + "checks": { + "kernels_straddling_decode_start": 0, + "host_gap_attribution_residual_ns": 0, + "idle_gap_before_first_decode_dispatch_us": 2233.4, + "last_dispatch_before_decode": "void mlx::core::rocm::arg_reduce_final, 256>(hip_bfloat16 const*, un" + }, + "top_kernels": [ + { + "kernel": "void mlx::core::rocm::qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int, int, bool)", + "class": "qmv", + "calls_per_token": 116.09, + "share_pct": 39.61 + }, + { + "kernel": "void mlx::core::rocm::gather_qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, unsigned int const*, unsigned int const*, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, hip_bfloat16*, int, int, int, int, int, bool, bool, long)", + "class": "gather_qmv", + "calls_per_token": 45.64, + "share_pct": 26.23 + }, + { + "kernel": "void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)", + "class": "binary", + "calls_per_token": 92.27, + "share_pct": 4.22 + }, + { + "kernel": "void mlx::core::rocm::binary_g(float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)", + "class": "binary", + "calls_per_token": 114.1, + "share_pct": 3.47 + }, + { + "kernel": "void mlx::core::rocm::gemv_batched_inline(float const*, float const*, float*, int, int, mlx::core::rocm::GemvBatchParams)", + "class": "gemm", + "calls_per_token": 45.64, + "share_pct": 2.23 + }, + { + "kernel": "void mlx::core::rocm::block_sort_kernel(float const*, unsigned int*, int, long, long, long, long)", + "class": "sort", + "calls_per_token": 22.82, + "share_pct": 2.15 + }, + { + "kernel": "void mlx::core::rocm::copy_v(hip_bfloat16 const*, float*, unsigned int)", + "class": "copy", + "calls_per_token": 206.38, + "share_pct": 2.01 + }, + { + "kernel": "void mlx::core::rocm::rms_norm_kernel(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, float, unsigned int, long)", + "class": "rms_norm", + "calls_per_token": 52.59, + "share_pct": 1.68 + }, + { + "kernel": "void mlx::core::rocm::gemv_single(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int)", + "class": "gemm", + "calls_per_token": 22.82, + "share_pct": 1.57 + }, + { + "kernel": "void mlx::core::rocm::binary_vv(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)", + "class": "binary", + "calls_per_token": 97.23, + "share_pct": 1.34 + } + ], + "class_share_pct": { + "qmv": 39.61, + "gather_qmv": 26.23, + "binary": 11.94, + "copy": 6.34, + "gemm": 5.03, + "sort": 2.15, + "compiled": 2.11, + "rms_norm": 2.01, + "unary": 1.1, + "ternary": 0.78, + "reduce": 0.76, + "scan": 0.59, + "sdpa": 0.51, + "conv": 0.38, + "gather_scatter": 0.36, + "arg_reduce": 0.08, + "dequantize": 0.01 + }, + "role_share_pct": { + "unattributed": 46.58, + "moe_expert_gemv": 26.23, + "ssm_step": 20.7, + "add_rms_join": 2.12, + "ssm_gated_norm": 1.17, + "ssm_conv": 0.95, + "ssm_silu": 0.88, + "moe_activation": 0.62, + "moe_gather_indices": 0.29, + "add_rms_join_post_attn": 0.27, + "sampler_tail": 0.19, + "rope_append": 0.0, + "moe_weighted_sum": 0.0, + "paged_attention": 0.0 + }, + "role_dispatches_per_token": { + "unattributed": 367.4, + "ssm_step": 1141.0, + "ssm_conv": 68.5, + "ssm_silu": 45.6, + "ssm_gated_norm": 91.3, + "add_rms_join": 91.3, + "moe_gather_indices": 45.6, + "moe_expert_gemv": 45.6, + "moe_activation": 45.6, + "add_rms_join_post_attn": 11.9, + "sampler_tail": 8.9, + "rope_append": 0.0, + "moe_weighted_sum": 0.0, + "paged_attention": 0.0 + }, + "role_host_gap_ms_per_token": { + "ssm_step": 6.7311, + "unattributed": 1.1722, + "ssm_gated_norm": 0.2126, + "add_rms_join": 0.1959, + "moe_activation": 0.1564, + "ssm_conv": 0.1551, + "ssm_silu": 0.116, + "moe_gather_indices": 0.1006, + "moe_expert_gemv": 0.0987, + "sampler_tail": 0.0433, + "add_rms_join_post_attn": 0.0256, + "tail": 0.0003 + }, + "port_units": { + "2063": { + "port": "fused_add_rms_norm + fused_rope_qk_append", + "fallback_share_pct": 0.27, + "fallback_dispatches_per_token": 11.9, + "reached_default_share_pct": 0.0, + "reached_default_dispatches_per_token": 0.0, + "reached_optin_share_pct": 0.0, + "fallback_host_gap_ms_per_token": 0.0256, + "fallback_host_gap_us_per_dispatch": 2.15, + "fallback_wall_share_pct": 0.27, + "ceiling_gpu_share": 1.0, + "ceiling_wall": 1.0, + "plain_ceiling_wall_est": 1.0, + "reached_default_host_gap_ms_per_token": 0.0, + "reached_default_host_gap_us_per_dispatch": 0.0, + "reached_default_wall_share_pct": 0.0, + "reached_default_ceiling_gpu_share": 1.0, + "reached_default_ceiling_wall": 1.0, + "reached_default_plain_ceiling_wall_est": 1.0, + "note": "nemotron_h never calls fused_add_rms_norm or forward_fused_rope_append" + }, + "2064": { + "port": "samplers (gumbel_max_sample, rejection_sample)", + "fallback_share_pct": 0.19, + "fallback_dispatches_per_token": 8.9, + "reached_default_share_pct": 0.0, + "reached_default_dispatches_per_token": 0.0, + "reached_optin_share_pct": 0.0, + "fallback_host_gap_ms_per_token": 0.0433, + "fallback_host_gap_us_per_dispatch": 4.85, + "fallback_wall_share_pct": 0.31, + "ceiling_gpu_share": 1.0, + "ceiling_wall": 1.0, + "plain_ceiling_wall_est": 1.0, + "reached_default_host_gap_ms_per_token": 0.0, + "reached_default_host_gap_us_per_dispatch": 0.0, + "reached_default_wall_share_pct": 0.0, + "reached_default_ceiling_gpu_share": 1.0, + "reached_default_ceiling_wall": 1.0, + "reached_default_plain_ceiling_wall_est": 1.0, + "note": "greedy argmax dispatches neither sampler kernel" + }, + "2065": { + "port": "fused MoE decode (moe_gateup, moe_down)", + "fallback_share_pct": 27.14, + "fallback_dispatches_per_token": 136.9, + "reached_default_share_pct": 0.0, + "reached_default_dispatches_per_token": 0.0, + "reached_optin_share_pct": 0.0, + "fallback_host_gap_ms_per_token": 0.3557, + "fallback_host_gap_us_per_dispatch": 2.6, + "fallback_wall_share_pct": 18.04, + "ceiling_gpu_share": 1.37, + "ceiling_wall": 1.22, + "plain_ceiling_wall_est": 1.26, + "reached_default_host_gap_ms_per_token": 0.0, + "reached_default_host_gap_us_per_dispatch": 0.0, + "reached_default_wall_share_pct": 0.0, + "reached_default_ceiling_gpu_share": 1.0, + "reached_default_ceiling_wall": 1.0, + "reached_default_plain_ceiling_wall_est": 1.0, + "note": "fused_moe_forward's default branch is gather_qmm; the kernel path needs MLXCEL_FUSED_MOE_RELU2 and moe_fc1_relu2 (#2069)" + }, + "2067": { + "port": "SSM update (ssm_update_kernel)", + "fallback_share_pct": 20.7, + "fallback_dispatches_per_token": 1141.0, + "reached_default_share_pct": 20.7, + "reached_default_dispatches_per_token": 1141.0, + "reached_optin_share_pct": 20.7, + "fallback_host_gap_ms_per_token": 6.7311, + "fallback_host_gap_us_per_dispatch": 5.9, + "fallback_wall_share_pct": 41.92, + "ceiling_gpu_share": 1.26, + "ceiling_wall": 1.72, + "plain_ceiling_wall_est": 1.56, + "reached_default_host_gap_ms_per_token": 6.7311, + "reached_default_host_gap_us_per_dispatch": 5.9, + "reached_default_wall_share_pct": 41.92, + "reached_default_ceiling_gpu_share": 1.26, + "reached_default_ceiling_wall": 1.72, + "reached_default_plain_ceiling_wall_est": 1.56, + "note": "ssm_kernel_available() gates the decode step" + }, + "2068": { + "port": "paged attention (v1, v2, merge)", + "fallback_share_pct": 0.0, + "fallback_dispatches_per_token": 0.0, + "reached_default_share_pct": 0.0, + "reached_default_dispatches_per_token": 0.0, + "reached_optin_share_pct": 0.0, + "fallback_host_gap_ms_per_token": 0.0, + "fallback_host_gap_us_per_dispatch": 0.0, + "fallback_wall_share_pct": 0.0, + "ceiling_gpu_share": 1.0, + "ceiling_wall": 1.0, + "plain_ceiling_wall_est": 1.0, + "reached_default_host_gap_ms_per_token": 0.0, + "reached_default_host_gap_us_per_dispatch": 0.0, + "reached_default_wall_share_pct": 0.0, + "reached_default_ceiling_gpu_share": 1.0, + "reached_default_ceiling_wall": 1.0, + "reached_default_plain_ceiling_wall_est": 1.0, + "note": "the bench decodes into a dense KVCache; the paged path is not taken" + } + } +} diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/ab_decode.txt b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/ab_decode.txt new file mode 100644 index 000000000..6eda01139 --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/ab_decode.txt @@ -0,0 +1,12 @@ +granite-4.0-h-tiny-4bit_ssm0_run1: Decode: 2453.27 ms (52.18 tok/s) +granite-4.0-h-tiny-4bit_ssm0_run2: Decode: 2103.56 ms (60.85 tok/s) +granite-4.0-h-tiny-4bit_ssm0_run3: Decode: 2137.75 ms (59.88 tok/s) +granite-4.0-h-tiny-4bit_ssm1_run1: Decode: 1431.27 ms (89.43 tok/s) +granite-4.0-h-tiny-4bit_ssm1_run2: Decode: 1432.90 ms (89.33 tok/s) +granite-4.0-h-tiny-4bit_ssm1_run3: Decode: 1431.84 ms (89.40 tok/s) +NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_ssm0_run1: Decode: 2430.48 ms (52.66 tok/s) +NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_ssm0_run2: Decode: 2458.77 ms (52.06 tok/s) +NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_ssm0_run3: Decode: 2447.76 ms (52.29 tok/s) +NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_ssm1_run1: Decode: 1705.68 ms (75.04 tok/s) +NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_ssm1_run2: Decode: 1700.74 ms (75.26 tok/s) +NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_ssm1_run3: Decode: 1698.26 ms (75.37 tok/s) diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/ab_guard.log b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/ab_guard.log new file mode 100644 index 000000000..b447b7341 --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/ab_guard.log @@ -0,0 +1,105 @@ +2026-10-08T23:14:10+0900 waiting for guard lock /tmp/mlxcel-rocm-gpu-guard.lock +2026-10-08T23:49:15+0900 acquired guard lock /tmp/mlxcel-rocm-gpu-guard.lock after 2105s +2026-10-08T23:49:15+0900 waiting for 90s of idle GPU and no compiler +2026-10-08T23:51:29+0900 idle for 90s: kfd proc empty and no compiler in every 1 Hz sample +2026-10-08T23:51:29+0900 attempt 1/5: start: i2148-ab.sh models/mlx/granite-4.0-h-tiny-4bit i2148-ab +2026-10-08T23:51:29+0900 sample 1: clean gpu=[] +2026-10-08T23:51:31+0900 sample 2: clean gpu=[3930415:mlxcel-bench-de] +2026-10-08T23:51:32+0900 sample 3: clean gpu=[3930415:mlxcel-bench-de] +2026-10-08T23:51:34+0900 sample 4: clean gpu=[3930415:mlxcel-bench-de] +2026-10-08T23:51:35+0900 sample 5: clean gpu=[3930415:mlxcel-bench-de] +2026-10-08T23:51:37+0900 sample 6: clean gpu=[3934372:mlxcel-bench-de] +2026-10-08T23:51:38+0900 sample 7: clean gpu=[3934372:mlxcel-bench-de] +2026-10-08T23:51:40+0900 sample 8: clean gpu=[3934372:mlxcel-bench-de] +2026-10-08T23:51:41+0900 sample 9: clean gpu=[3937766:mlxcel-bench-de] +2026-10-08T23:51:43+0900 sample 10: clean gpu=[3937766:mlxcel-bench-de] +2026-10-08T23:51:44+0900 sample 11: clean gpu=[3937766:mlxcel-bench-de] +2026-10-08T23:51:45+0900 sample 12: clean gpu=[3937766:mlxcel-bench-de] +2026-10-08T23:51:47+0900 sample 13: clean gpu=[3940947:mlxcel-bench-de] +2026-10-08T23:51:48+0900 sample 14: clean gpu=[3940947:mlxcel-bench-de] +2026-10-08T23:51:50+0900 sample 15: clean gpu=[3940947:mlxcel-bench-de] +2026-10-08T23:51:51+0900 sample 16: clean gpu=[3944448:mlxcel-bench-de] +2026-10-08T23:51:53+0900 sample 17: clean gpu=[3944448:mlxcel-bench-de] +2026-10-08T23:51:54+0900 sample 18: clean gpu=[3944448:mlxcel-bench-de] +2026-10-08T23:51:56+0900 sample 19: clean gpu=[3944448:mlxcel-bench-de] +2026-10-08T23:51:57+0900 sample 20: clean gpu=[3947539:mlxcel-bench-de] +2026-10-08T23:51:59+0900 sample 21: clean gpu=[3947539:mlxcel-bench-de] +2026-10-08T23:52:00+0900 sample 22: clean gpu=[3947539:mlxcel-bench-de] +2026-10-08T23:52:01+0900 monitor: 22 samples +2026-10-08T23:52:01+0900 attempt 1: CLEAN, exit 0 +2026-10-08T23:52:01+0900 waiting for guard lock /tmp/mlxcel-rocm-gpu-guard.lock +2026-10-09T00:52:15+0900 acquired guard lock /tmp/mlxcel-rocm-gpu-guard.lock after 3614s +2026-10-09T00:52:15+0900 waiting for 90s of idle GPU and no compiler +2026-10-09T00:53:01+0900 idle streak reset after 31s: gpu=[] compilers=[2060323:cargo] +2026-10-09T00:55:23+0900 idle for 90s: kfd proc empty and no compiler in every 1 Hz sample +2026-10-09T00:55:23+0900 attempt 1/5: start: i2148-ab.sh models/mlx/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit i2148-ab +2026-10-09T00:55:24+0900 sample 1: clean gpu=[] +2026-10-09T00:55:25+0900 sample 2: clean gpu=[2149976:mlxcel-bench-de] +2026-10-09T00:55:27+0900 sample 3: clean gpu=[2149976:mlxcel-bench-de] +2026-10-09T00:55:28+0900 sample 4: clean gpu=[2149976:mlxcel-bench-de] +2026-10-09T00:55:30+0900 sample 5: clean gpu=[2149976:mlxcel-bench-de] +2026-10-09T00:55:31+0900 sample 6: clean gpu=[2149976:mlxcel-bench-de] +2026-10-09T00:55:33+0900 sample 7: clean gpu=[2149976:mlxcel-bench-de] +2026-10-09T00:55:34+0900 sample 8: clean gpu=[2149976:mlxcel-bench-de] +2026-10-09T00:55:36+0900 sample 9: clean gpu=[2149976:mlxcel-bench-de] +2026-10-09T00:55:37+0900 sample 10: clean gpu=[2149976:mlxcel-bench-de] +2026-10-09T00:55:39+0900 sample 11: clean gpu=[2149976:mlxcel-bench-de] +2026-10-09T00:55:40+0900 sample 12: clean gpu=[] +2026-10-09T00:55:42+0900 sample 13: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:43+0900 sample 14: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:45+0900 sample 15: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:46+0900 sample 16: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:48+0900 sample 17: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:49+0900 sample 18: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:51+0900 sample 19: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:52+0900 sample 20: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:54+0900 sample 21: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:55+0900 sample 22: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:57+0900 sample 23: clean gpu=[2160431:mlxcel-bench-de] +2026-10-09T00:55:58+0900 sample 24: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:00+0900 sample 25: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:01+0900 sample 26: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:03+0900 sample 27: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:04+0900 sample 28: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:06+0900 sample 29: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:07+0900 sample 30: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:09+0900 sample 31: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:10+0900 sample 32: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:12+0900 sample 33: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:13+0900 sample 34: clean gpu=[2172055:mlxcel-bench-de] +2026-10-09T00:56:15+0900 sample 35: clean gpu=[2182099:mlxcel-bench-de] +2026-10-09T00:56:16+0900 sample 36: clean gpu=[2182099:mlxcel-bench-de] +2026-10-09T00:56:18+0900 sample 37: clean gpu=[2182099:mlxcel-bench-de] +2026-10-09T00:56:19+0900 sample 38: clean gpu=[2182099:mlxcel-bench-de] +2026-10-09T00:56:21+0900 sample 39: clean gpu=[2182099:mlxcel-bench-de] +2026-10-09T00:56:22+0900 sample 40: clean gpu=[2182099:mlxcel-bench-de] +2026-10-09T00:56:24+0900 sample 41: clean gpu=[2182099:mlxcel-bench-de] +2026-10-09T00:56:25+0900 sample 42: clean gpu=[2182099:mlxcel-bench-de] +2026-10-09T00:56:27+0900 sample 43: clean gpu=[2182099:mlxcel-bench-de] +2026-10-09T00:56:28+0900 sample 44: clean gpu=[2182099:mlxcel-bench-de] +2026-10-09T00:56:30+0900 sample 45: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:31+0900 sample 46: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:33+0900 sample 47: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:34+0900 sample 48: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:36+0900 sample 49: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:37+0900 sample 50: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:39+0900 sample 51: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:40+0900 sample 52: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:42+0900 sample 53: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:43+0900 sample 54: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:45+0900 sample 55: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:46+0900 sample 56: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:48+0900 sample 57: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:49+0900 sample 58: clean gpu=[2192279:mlxcel-bench-de] +2026-10-09T00:56:51+0900 sample 59: clean gpu=[2192279:?] +2026-10-09T00:56:52+0900 sample 60: clean gpu=[2205724:mlxcel-bench-de] +2026-10-09T00:56:54+0900 sample 61: clean gpu=[2205724:mlxcel-bench-de] +2026-10-09T00:56:55+0900 sample 62: clean gpu=[2205724:mlxcel-bench-de] +2026-10-09T00:56:57+0900 sample 63: clean gpu=[2205724:mlxcel-bench-de] +2026-10-09T00:56:58+0900 sample 64: clean gpu=[2205724:mlxcel-bench-de] +2026-10-09T00:57:00+0900 sample 65: clean gpu=[2205724:mlxcel-bench-de] +2026-10-09T00:57:01+0900 sample 66: clean gpu=[2205724:mlxcel-bench-de] +2026-10-09T00:57:03+0900 sample 67: clean gpu=[2205724:mlxcel-bench-de] +2026-10-09T00:57:04+0900 sample 68: clean gpu=[2205724:mlxcel-bench-de] +2026-10-09T00:57:05+0900 monitor: 68 samples +2026-10-09T00:57:05+0900 attempt 1: CLEAN, exit 0 diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_bench.log b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_bench.log new file mode 100644 index 000000000..b28513b3d --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_bench.log @@ -0,0 +1,21 @@ +[mlx-rocm] bound HIP device 0: gfx1151 (AMD Radeon 8060S Graphics) cus=20 warp=32 lds=64KB +[Memory] cache limit: 2.15 GB +[Load] wall: 0.112 s MLX peak: 0.00 GB +[Memory] after load: active 0.00 GB, cache 0.00 GB, active+cache 0.00 GB, phase peak 0.00 GB +[long-prompt] target=512 tokens -> using 512 tokens (max_context=Some(131072), reserved 128 for generation) +[phase] warmup_start monotonic_ns=867765408366292 boottime_ns=867765408366332 +[mlx-rocm] gfx1151 native_wmma=1 cus=20 warp=32 low_cu=0 host_WARP_SIZE=32 +[hipBLASLt caps] device 0 (gfx1151): bf16=1 fp8_e4m3=0 fp8_e5m2=0 int8=1 +[Memory] after warmup (prefill + warmup tokens): active 3.96 GB, cache 1.77 GB, active+cache 5.72 GB, phase peak 7.22 GB +[long-prompt] target=512 tokens -> using 512 tokens (max_context=Some(131072), reserved 128 for generation) +[phase] measured_start monotonic_ns=867768017516044 boottime_ns=867768017516064 +[phase] decode_start monotonic_ns=867768855957105 boottime_ns=867768855957135 +[phase] measured_end monotonic_ns=867771334494191 boottime_ns=867771334494221 +[Memory] after measured pass: active 3.96 GB, cache 1.77 GB, active+cache 5.73 GB, phase peak 7.31 GB +[Profile Results] + Prompt tokens: 512 + Generated tokens: 128 + Prefill: 829.08 ms (617.55 tok/s) + Decode: 2478.54 ms (51.64 tok/s) + Decode path: engine + MLX peak memory: 7.31 GB diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_decode_kernels.csv b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_decode_kernels.csv new file mode 100644 index 000000000..aa6691312 --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_decode_kernels.csv @@ -0,0 +1,55 @@ +kernel,class,roles,port_units,calls,calls_per_token,total_ns,avg_ns,share_pct +"void mlx::core::rocm::qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int, int, bool)",qmv,,,21463,167.68,352286737,16414,25.183 +"void mlx::core::rocm::gather_qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, unsigned int const*, unsigned int const*, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, hip_bfloat16*, int, int, int, int, int, bool, bool, long)",gather_qmv,moe_expert_gemv,2065,15240,119.06,264421645,17351,18.902 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,18252,142.59,96341851,5278,6.887 +"void mlx::core::rocm::binary_g(float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",binary,ssm_step,2067,22824,178.31,80755831,3538,5.773 +"void mlx::core::rocm::block_sort_kernel(hip_bfloat16 const*, unsigned int*, int, long, long, long, long)",sort,,,5080,39.69,52150941,10266,3.728 +"void mlx::core::rocm::rms_norm_kernel(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, float, unsigned int, long)",rms_norm,add_rms_join;ssm_gated_norm,,14859,116.09,50073191,3370,3.579 +"void mlx::core::rocm::copy_v(hip_bfloat16 const*, float*, unsigned int)",copy,moe_weighted_sum;sampler_tail;ssm_step,2064;2065;2067,42291,330.4,40669681,962,2.907 +"void mlx::core::rocm::CV2ISigmoidADV2IBroadcastACEV2IBroadcastCAFV2IMultiplyDEGV2IBroadcastFBHV2IBroadcastBFIV2OMultiplyGH_VV_V2V2_6142509188972423790_contiguous(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",compiled,moe_activation,2065,10160,79.38,40541125,3990,2.898 +"void mlx::core::rocm::binary_vv(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",binary,add_rms_join;ssm_step,2067,19812,154.78,35824059,1808,2.561 +"void mlx::core::rocm::BV2ISigmoidACV2IBroadcastABDV2IBroadcastBAEV2OMultiplyCD_V_V2_6142509188972423790_contiguous(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",compiled,ssm_silu,,9144,71.44,26553466,2904,1.898 +"void mlx::core::rocm::gemv_batched_inline(float const*, float const*, float*, int, int, mlx::core::rocm::GemvBatchParams)",gemm,ssm_step,2067,4572,35.72,23185131,5071,1.657 +"void mlx::core::rocm::binary_vs(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",binary,,,10287,80.37,19648303,1910,1.405 +Cijk_Alik_Bjlk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8,gemm,ssm_step,2067,4536,35.44,19119782,4215,1.367 +"void mlx::core::rocm::ternary_g(bool const*, float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",ternary,ssm_step,2067,9144,71.44,18564917,2030,1.327 +"void mlx::core::rocm::binary_vv(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",binary,ssm_gated_norm;ssm_step,2067,9144,71.44,17085620,1869,1.221 +"void mlx::core::rocm::copy_v(float const*, hip_bfloat16*, unsigned int)",copy,moe_activation;sampler_tail;ssm_step,2064;2065;2067,14859,116.09,16957631,1141,1.212 +"void mlx::core::rocm::copy_s(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",copy,ssm_gated_norm;ssm_step,2067,14994,117.14,16225812,1082,1.160 +"void mlx::core::rocm::arange_kernel(int*, int, int, unsigned long)",copy,ssm_step,2067,18288,142.88,14823469,811,1.060 +"void mlx::core::rocm::qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int, int, bool)",qmv,,,5080,39.69,14808967,2915,1.059 +"void mlx::core::rocm::copy_s(float const*, float*, unsigned int)",copy,ssm_step,2067,13716,107.16,14515117,1058,1.038 +"void mlx::core::rocm::unary_v(float const*, float*, unsigned int)",unary,ssm_step,2067,9144,71.44,14513812,1587,1.038 +"void mlx::core::rocm::copy_gg_byval(hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",copy,ssm_conv,,10176,79.5,12549078,1233,0.897 +"void mlx::core::rocm::softmax_kernel(float const*, float*, int)",softmax,,,5080,39.69,11204754,2206,0.801 +"void mlx::core::rocm::binary_ss(int const*, int const*, bool*, unsigned int)",binary,ssm_step,2067,9144,71.44,10510842,1149,0.751 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,4572,35.72,9851892,2155,0.704 +"void mlx::core::rocm::col_reduce_small(float const*, float*, mlx::core::rocm::ColReduceArgs, unsigned long)",reduce,,,5080,39.69,9151365,1801,0.654 +"void mlx::core::rocm::arange_kernel(unsigned int*, unsigned int, unsigned int, unsigned long)",copy,moe_gather_indices,2065,10160,79.38,8076713,795,0.577 +"void mlx::core::(anonymous namespace)::depthwise_conv1d_kernel(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, mlx::core::ConvParams<1>)",conv,ssm_conv,,4572,35.72,8063061,1764,0.576 +Cijk_Ailk_Bljk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8,gemm,ssm_step,2067,4572,35.72,7843484,1716,0.561 +"void mlx::core::rocm::gather_axis_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long)",gather_scatter,sampler_tail,2064,5207,40.68,7705332,1480,0.551 +"void mlx::core::rocm::binary_vs(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,4572,35.72,7312901,1599,0.523 +"void mlx::core::rocm::binary_vs(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,4572,35.72,7250566,1586,0.518 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,4572,35.72,7111092,1555,0.508 +"void mlx::core::rocm::unary_v(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",unary,ssm_step,2067,4572,35.72,7051964,1542,0.504 +"void mlx::core::rocm::binary_g(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",binary,moe_weighted_sum,2065,5080,39.69,7040461,1386,0.503 +"void mlx::core::rocm::contiguous_scan(float const*, float*, int)",scan,ssm_step,2067,4572,35.72,6742998,1475,0.482 +"void mlx::core::rocm::strided_scan(float const*, float*, int, long, long)",scan,ssm_step,2067,4572,35.72,6477178,1417,0.463 +"void mlx::core::rocm::unary_v(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",unary,ssm_step,2067,4572,35.72,6176524,1351,0.442 +"void mlx::core::rocm::kernel_sdpav_1pass(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, hip_bfloat16 const*, mlx::core::rocm::AttnParams)",sdpa,,,508,3.97,6085243,11979,0.435 +"void mlx::core::rocm::binary_sv(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,4572,35.72,5616271,1228,0.401 +"void mlx::core::rocm::copy_g_byval(float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",copy,ssm_step,2067,4572,35.72,5594668,1224,0.400 +"void mlx::core::rocm::gemv_single(float const*, float const*, float*, int, int)",gemm,ssm_step,2067,4572,35.72,5060858,1107,0.362 +Cijk_Ailk_Bjlk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8,gemm,ssm_step,2067,36,0.28,4203591,116766,0.300 +"void mlx::core::rocm::arg_reduce_partial, 256>(hip_bfloat16 const*, hip_bfloat16*, unsigned int*, unsigned long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, long, int, int)",arg_reduce,sampler_tail,2064,127,0.99,714618,5627,0.051 +"void mlx::core::rocm::binary_vs(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",binary,sampler_tail,2064,127,0.99,647014,5095,0.046 +"void mlx::core::rocm::gather_rows_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, unsigned int, int)",gather_scatter,,,254,1.98,345477,1360,0.025 +"void mlx::core::rocm::arg_reduce_final, 256>(hip_bfloat16 const*, unsigned int const*, unsigned int*, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, unsigned long, int)",arg_reduce,sampler_tail,2064,127,0.99,252876,1991,0.018 +"void mlx::core::rocm::scatter_axis_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long, long)",gather_scatter,sampler_tail,2064,127,0.99,219571,1729,0.016 +"void mlx::core::rocm::copy_v(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",copy,sampler_tail,2064,127,0.99,217283,1711,0.016 +"void mlx::core::rocm::affine_dequantize_packed_kernel(unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned long, int)",dequantize,,,127,0.99,191801,1510,0.014 +"void mlx::core::rocm::gather_rows_kernel(unsigned int const*, int const*, unsigned int*, long, unsigned int, int)",gather_scatter,,,127,0.99,178417,1405,0.013 +"void mlx::core::rocm::copy_v(unsigned int const*, int*, unsigned int)",copy,sampler_tail,2064,126,0.98,172325,1368,0.012 +"void mlx::core::rocm::binary_ss(float const*, float const*, float*, unsigned int)",binary,sampler_tail,2064,127,0.99,157860,1243,0.011 +"void mlx::core::rocm::copy_g_byval(hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",copy,ssm_step,2067,36,0.28,66607,1850,0.005 diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_kernel_stats.csv b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_kernel_stats.csv new file mode 100644 index 000000000..fa3182dae --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_kernel_stats.csv @@ -0,0 +1,82 @@ +"Name","Calls","TotalDurationNs","AverageNs","Percentage","MinNs","MaxNs","StdDev" +"void mlx::core::rocm::gather_qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, unsigned int const*, unsigned int const*, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, hip_bfloat16*, int, int, int, int, int, bool, bool, long)",17760,1388293949,78169.704336,42.70,14787,6889820,547195.867906 +"void mlx::core::rocm::qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int, int, bool)",24674,405481898,16433.569668,12.47,3486,429805,30289.600168 +"Cijk_Ailk_Bljk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8",5328,170920751,32079.720533,5.26,1442,2739128,260437.652040 +"void mlx::core::rocm::binary_g(float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",26496,144153288,5440.567935,4.43,1002,616955,28942.454270 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",20952,110252405,5262.142278,3.39,1242,73818,6194.621480 +"void mlx::core::rocm::ternary_g(bool const*, float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",10658,79516316,7460.716457,2.45,1523,875199,51002.638306 +"void mlx::core::rocm::rms_norm_kernel(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, float, unsigned int, long)",17316,61455968,3549.085701,1.89,2925,57748,1735.285050 +"void mlx::core::rocm::block_sort_kernel(hip_bfloat16 const*, unsigned int*, int, long, long, long, long)",5920,61443360,10378.945946,1.89,7373,70091,1882.498549 +"void mlx::core::rocm::strided_scan(float const*, float*, int, long, long)",5328,59523215,11171.774587,1.83,1282,2570332,89783.638970 +"void mlx::core::rocm::copy_v(hip_bfloat16 const*, float*, unsigned int)",48996,50616609,1033.076353,1.56,721,143469,1995.376446 +"void mlx::core::rocm::unary_v(float const*, float*, unsigned int)",10584,49491410,4676.059146,1.52,1322,972421,37892.140787 +"void mlx::core::rocm::CV2ISigmoidADV2IBroadcastACEV2IBroadcastCAFV2IMultiplyDEGV2IBroadcastFBHV2IBroadcastBFIV2OMultiplyGH_VV_V2V2_6142509188972423790_contiguous(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",11760,48843201,4153.333418,1.50,2965,33623,2062.091456 +"void mlx::core::rocm::binary_vv(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",23016,47033147,2043.497871,1.45,1402,109606,2452.842780 +"Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT96x128x32_MI16x16x1_SN_LDSB0_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS1_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSTI0_LBSPPA128_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT3_4_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR1_PLR0_PKA0_SGROB0_SIA3_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB4_WSGRA0_WSGRB0_WS32_WG32_4_1_WGMXCC1",74,33878675,457819.932432,1.04,259205,9340530,1113927.670432 +"void mlx::core::rocm::BV2ISigmoidACV2IBroadcastABDV2IBroadcastBAEV2OMultiplyCD_V_V2_6142509188972423790_contiguous(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",10584,31980719,3021.609883,0.9836,2524,38071,1486.613312 +"void mlx::core::rocm::affine_dequantize_packed_kernel(unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned long, int)",486,28806102,59271.814815,0.8859,1322,3089224,184997.375754 +"void mlx::core::rocm::gemv_batched_inline(float const*, float const*, float*, int, int, mlx::core::rocm::GemvBatchParams)",5256,26649937,5070.383752,0.8196,4529,23003,469.305408 +"void mlx::core::rocm::binary_vs(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",11988,26592749,2218.280697,0.8179,1442,136094,3124.497475 +"void mlx::core::rocm::col_reduce_small(float const*, float*, mlx::core::rocm::ColReduceArgs, unsigned long)",5920,25807839,4359.432264,0.7937,1522,787474,25920.700755 +"void mlx::core::rocm::binary_vv(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",10584,22877633,2161.529951,0.7036,1523,49573,3556.391084 +"void mlx::core::rocm::copy_v(float const*, hip_bfloat16*, unsigned int)",17324,21875868,1262.749250,0.6728,881,36588,1346.406847 +"Cijk_Alik_Bjlk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8",5184,21848406,4214.584491,0.6719,3447,19837,421.012888 +"void mlx::core::rocm::binary_g(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",5992,21818681,3641.301903,0.6710,1162,140744,15656.644372 +"void mlx::core::rocm::copy_s(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",17552,19217955,1094.915394,0.5910,881,8015,397.917059 +"Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT128x64x32_MI16x16x1_SN_LDSB0_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR0_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT2_4_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR0_PLR0_PKA0_SGROB0_SIA3_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSM1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS32_WG64_2_1_WGMXCC1",168,18322342,109061.559524,0.5635,52298,381915,53656.255269 +"void mlx::core::rocm::copy_gg_byval(hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",11856,18169974,1532.555162,0.5588,881,206627,3989.699206 +"void mlx::core::rocm::arange_kernel(int*, int, int, unsigned long)",21316,17420668,817.257834,0.5358,601,10019,416.660325 +"void mlx::core::rocm::copy_s(float const*, float*, unsigned int)",15984,17044310,1066.335711,0.5242,881,9297,375.538728 +"void mlx::core::rocm::qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int, int, bool)",5840,17020075,2914.396404,0.5235,2445,10099,312.133453 +"void mlx::core::(anonymous namespace)::depthwise_conv1d_kernel(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, mlx::core::ConvParams<1>)",5328,14993011,2814.003566,0.4611,1563,153688,9049.454652 +"void mlx::core::rocm::softmax_kernel(float const*, float*, int)",5920,13515634,2283.046284,0.4157,1923,12463,748.774449 +"void mlx::core::rocm::binary_ss(int const*, int const*, bool*, unsigned int)",10512,12081300,1149.286530,0.3716,881,9538,330.750726 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",5328,11640689,2184.814002,0.3580,1843,9538,439.037040 +"Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT128x64x64_MI16x16x1_SN_LDSB0_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR0_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSTI0_LBSPPA128_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT4_2_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA0_SGROB0_SIA1_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSM1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS32_WG32_4_1_WGMXCC1",80,9819218,122740.225000,0.3020,107000,306974,37735.344570 +"void mlx::core::rocm::arange_kernel(unsigned int*, unsigned int, unsigned int, unsigned long)",11760,9448277,803.424915,0.2906,601,14146,389.705386 +"void mlx::core::rocm::gather_axis_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long)",5988,8888033,1484.307448,0.2734,1202,10379,430.653116 +"void mlx::core::rocm::binary_vs(float const*, float const*, float*, unsigned int)",5328,8666542,1626.603228,0.2665,1402,18394,492.169259 +"void mlx::core::rocm::binary_vs(float const*, float const*, float*, unsigned int)",5328,8589948,1612.227477,0.2642,1322,8336,391.102014 +"Cijk_Ailk_Bjlk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8",72,8474106,117695.916667,0.2606,111850,174046,7440.888107 +"void mlx::core::rocm::unary_v(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",5328,8270106,1552.197072,0.2543,1242,10019,338.327594 +"void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)",5256,8174119,1555.197679,0.2514,1243,6813,306.467528 +"void mlx::core::rocm::contiguous_scan(float const*, float*, int)",5256,7755852,1475.618721,0.2385,1322,8697,310.824102 +"void mlx::core::rocm::unary_v(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",5328,7265161,1363.581269,0.2234,1202,8576,362.396021 +"void mlx::core::rocm::gather_rows_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, unsigned int, int)",456,7107903,15587.506579,0.2186,962,140103,21918.501386 +"void mlx::core::rocm::kernel_sdpav_1pass(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, hip_bfloat16 const*, mlx::core::rocm::AttnParams)",584,6921608,11852.068493,0.2129,7213,39194,1417.329997 +"void mlx::core::rocm::binary_sv(float const*, float const*, float*, unsigned int)",5256,6458607,1228.806507,0.1986,961,7013,366.856975 +"void mlx::core::rocm::copy_g_byval(float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",5256,6434570,1224.233257,0.1979,1082,8616,332.238475 +"void mlx::core::rocm::gemv_single(float const*, float const*, float*, int, int)",5256,5841241,1111.347222,0.1796,881,9017,357.198586 +"void mlx::core::rocm::binary_g(float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",144,5211075,36188.020833,0.1603,1603,90009,34148.429210 +"void mlx::core::rocm::copy_g_byval(hip_bfloat16 const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",288,4431379,15386.732639,0.1363,1643,175168,23561.777029 +"void mlx::core::rocm::binary_g(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",80,4179550,52244.375000,0.1285,46206,86883,11216.270928 +"void mlx::core::rocm::binary_vs(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",148,3495307,23616.939189,0.1075,4689,1415971,158928.866263 +"void mlx::core::rocm::block_sort_kernel(unsigned int const*, unsigned int*, int, long, long, long, long)",160,2566527,16040.793750,0.0789,13305,78827,6086.055659 +"Cijk_Alik_Bljk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8",72,2116003,29388.930556,0.0651,26249,129883,12139.172811 +"void mlx::core::rocm::BV2ISigmoidACV2IBroadcastABDV2IBroadcastBAEV2OMultiplyCD_V_V2_6142509188972423790_strided<2, unsigned int, 16>(hip_bfloat16 const*, hip::std::array, hip_bfloat16*, hip::std::array, unsigned int)",72,1905043,26458.930556,0.0586,24085,31659,1619.373156 +"void mlx::core::rocm::CV2ISigmoidADV2IBroadcastACEV2IBroadcastCAFV2IMultiplyDEGV2IBroadcastFBHV2IBroadcastBFIV2OMultiplyGH_VV_V2V2_6142509188972423790_strided<2, unsigned int, 16>(hip_bfloat16 const*, hip::std::array, hip_bfloat16 const*, hip::std::array, hip_bfloat16*, hip::std::array, unsigned int)",80,1381103,17263.787500,0.0425,13105,80030,8664.752760 +"void mlx::core::rocm::binary_g(int const*, int const*, bool*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",144,1332686,9254.763889,0.0410,7815,37310,3067.502768 +"Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT32x16x128_MI16x16x1_SN_LDSB0_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR0_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSTI0_LBSPPA256_LBSPPB256_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT1_1_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR1_PLR1_PKA0_SGROB0_SIA1_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSM1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS32_WG32_2_1_WGMXCC1",80,975463,12193.287500,0.0300,10019,44924,4095.709086 +"void mlx::core::rocm::copy_g_byval(hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",96,844223,8793.989583,0.0260,1443,36308,12188.159008 +"void mlx::core::rocm::arg_reduce_partial, 256>(hip_bfloat16 const*, hip_bfloat16*, unsigned int*, unsigned long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, long, int, int)",148,836287,5650.587838,0.0257,5130,11581,625.868473 +"Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT32x32x64_MI16x16x1_SN_LDSB1_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR0_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS1_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU4_K1_LDSTI0_LBSPPA256_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB8_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT2_1_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR1_PLR1_PKA0_SGROB0_SIA2_SS0_SPO0_SRVW0_SSO0_SVW2_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA2_VWB1_WSGRA0_WSGRB0_WS32_WG16_4_1_WGMXCC1",16,659714,41232.125000,0.0203,36468,45805,2586.235665 +"void mlx::core::rocm::softmax_kernel(hip_bfloat16 const*, hip_bfloat16*, int)",8,641841,80230.125000,0.0197,78907,81632,958.852206 +"Cijk_Ailk_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT64x112x32_MI16x16x1_SN_LDSB0_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR1_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS0_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB4_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU1_K1_LDSTI0_LBSPPA2048_LBSPPB128_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA16_LPB16_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT1_7_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR2_PLR1_PKA0_SGROB0_SIA3_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSM1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS32_WG64_2_1_WGMXCC1",8,510516,63814.500000,0.0157,61595,65763,1184.830187 +"Cijk_Alik_Bljk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT64x32x32_MI16x16x1_SN_LDSB1_AFC1_AG0_AGGSUA0_AGNTAB0_AFEM1_AFEM1_ASEM1_BL1_BS1_CD1_1_CLR0_CLS0_CADS0_DTLA0_DTLB0_DTLM0_DTVA0_DTVB0_DTVMXSA0_DTVMXSB0_DTVSM0_DPLB0_EPS1_ELFLR0_EMLLn1_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_HPLR0_ISA1151_ICIW0_IU2_K1_LDSTI0_LBSPPA256_LBSPPB0_LBSPPMXSA0_LBSPPMXSB0_LBSPPM0_LPA8_LPB0_LPMXSA0_LPMXSB0_LPM0_LRVW16_LWPMn1_MIAV1_MIWT4_1_MXLIBL_MXSFNS_MO40_MGRIPM1_NTn1_NTA0_NTB0_NTC0_NTD0_NTE0_NTMXSA0_NTMXSB0_NTM0_NTWS0_NVn1_NVA0_NVB0_NVC0_NVD0_NVE0_NVMXSA0_NVMXSB0_NVM0_NVWS0_NEPBS0_NLCA1_NLCB1_ONLL1_PAP0_PGL0_PGR1_PLR1_PKA0_SGROB0_SIA2_SS1_SPO0_SRVW0_SSO0_SVW4_SK0_SKFTR0_SKFDPO0_SKWS0_SKXCCM0_SNLL0_SIP1_SGRO0_TDMI0_TDMIM0_TDMS0_TIN0_THn1_THA0_THB0_THC0_THD0_THE0_THMXSA0_THMXSB0_THM0_THWS0_TLDS1_TLDSMn1_ULSGRO0_USL1_USLMX0_UDFMAC0_UIOFGRO0_UPLRP0_USFGROn1_USI0_VSn1_VWA4_VWB1_WSGRA0_WSGRB0_WS32_WG16_4_1_WGMXCC1",8,445434,55679.250000,0.0137,52859,59992,2486.196507 +"void mlx::core::rocm::affine_dequantize_packed_kernel(unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned long, int)",80,331622,4145.275000,0.0102,3326,14788,1660.902370 +"void mlx::core::rocm::arg_reduce_final, 256>(hip_bfloat16 const*, unsigned int const*, unsigned int*, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, unsigned long, int)",148,294794,1991.851351,9.066e-03,1843,2404,105.880074 +"void mlx::core::rocm::gather_general_kernel(unsigned int const*, unsigned int*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, mlx::core::rocm::hip_array, unsigned int, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",80,280808,3510.100000,8.636e-03,2043,9017,2182.533324 +"void mlx::core::rocm::scatter_axis_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long, long)",148,256682,1734.337838,7.894e-03,1483,6652,438.887687 +"void mlx::core::rocm::copy_v(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",148,256473,1732.925676,7.888e-03,1563,5169,307.802815 +"void mlx::core::rocm::gather_axis_kernel(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, int, long, long)",80,252391,3154.887500,7.762e-03,2444,9418,1464.324599 +"void mlx::core::rocm::gather_rows_kernel(unsigned int const*, int const*, unsigned int*, long, unsigned int, int)",148,240412,1624.405405,7.394e-03,1162,23283,1987.969688 +"void mlx::core::rocm::binary_vs(float const*, float const*, float*, unsigned int)",80,201299,2516.237500,6.191e-03,1923,11662,1529.436301 +"void mlx::core::rocm::copy_v(unsigned int const*, int*, unsigned int)",144,194807,1352.826389,5.991e-03,1082,9618,744.183180 +"void mlx::core::rocm::binary_ss(float const*, float const*, float*, unsigned int)",148,189718,1281.878378,5.835e-03,1082,6733,470.157680 +"void mlx::core::rocm::binary_sv(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",8,168997,21124.625000,5.197e-03,20599,21841,417.159421 +"void mlx::core::rocm::copy_g_byval(unsigned int const*, unsigned int*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",80,154725,1934.062500,4.759e-03,1242,8054,1544.784696 +"void mlx::core::rocm::copy_v(int const*, float*, unsigned int)",80,143352,1791.900000,4.409e-03,1082,8216,1563.680376 +"void mlx::core::rocm::copy_v(float const*, int*, unsigned int)",80,122990,1537.375000,3.783e-03,961,6492,1382.325626 +"void mlx::core::rocm::copy_v(unsigned int const*, float*, unsigned int)",80,95499,1193.737500,2.937e-03,1002,6212,583.480690 +"__amd_rocclr_fillBufferUnAligned",1,70853,70853.000000,2.179e-03,70853,70853,0.00000000e+00 +"void mlx::core::rocm::binary_g(int const*, int const*, bool*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)",2,17192,8596.000000,5.287e-04,8496,8696,141.421356 diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_plain_bench.log b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_plain_bench.log new file mode 100644 index 000000000..cd38577ce --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_plain_bench.log @@ -0,0 +1,17 @@ +[mlx-rocm] bound HIP device 0: gfx1151 (AMD Radeon 8060S Graphics) cus=20 warp=32 lds=64KB +[Memory] cache limit: 2.15 GB +[Load] wall: 0.142 s MLX peak: 0.00 GB +[Memory] after load: active 0.00 GB, cache 0.00 GB, active+cache 0.00 GB, phase peak 0.00 GB +[long-prompt] target=512 tokens -> using 512 tokens (max_context=Some(131072), reserved 128 for generation) +[mlx-rocm] gfx1151 native_wmma=1 cus=20 warp=32 low_cu=0 host_WARP_SIZE=32 +[hipBLASLt caps] device 0 (gfx1151): bf16=1 fp8_e4m3=0 fp8_e5m2=0 int8=1 +[Memory] after warmup (prefill + warmup tokens): active 3.96 GB, cache 1.92 GB, active+cache 5.87 GB, phase peak 7.22 GB +[long-prompt] target=512 tokens -> using 512 tokens (max_context=Some(131072), reserved 128 for generation) +[Memory] after measured pass: active 3.96 GB, cache 1.92 GB, active+cache 5.87 GB, phase peak 7.31 GB +[Profile Results] + Prompt tokens: 512 + Generated tokens: 128 + Prefill: 867.35 ms (590.30 tok/s) + Decode: 2102.50 ms (60.88 tok/s) + Decode path: engine + MLX peak memory: 7.31 GB diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_summary.json b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_summary.json new file mode 100644 index 000000000..4dd7ae47b --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/granite-4.0-h-tiny-4bit_greedy_summary.json @@ -0,0 +1,264 @@ +{ + "name": "granite-4.0-h-tiny-4bit_greedy", + "model_type": "granitemoehybrid", + "temperature": 0.0, + "clock": "boottime", + "prompt_tokens": 512, + "generated_tokens": 128, + "profiled_decode_tok_s": 51.64, + "profiled_prefill_tok_s": 617.55, + "plain_decode_tok_s": 60.88, + "plain_prefill_tok_s": 590.3, + "profiler_decode_slowdown_pct": 17.89, + "decode_dispatches": 404228, + "dispatches_per_token": 3158.0, + "decode_wall_ms": 2478.537, + "decode_gpu_sum_ms": 1398.912, + "decode_gpu_busy_ms": 1398.911, + "gpu_ms_per_token": 10.929, + "host_gap_ms_per_token": 8.4346, + "host_gap_pct_of_wall": 43.56, + "plain_wall_ms_per_token": 16.4258, + "plain_host_gap_ms_per_token_est": 5.4968, + "checks": { + "kernels_straddling_decode_start": 0, + "host_gap_attribution_residual_ns": 0, + "idle_gap_before_first_decode_dispatch_us": 13781.6, + "last_dispatch_before_decode": "void mlx::core::rocm::arg_reduce_final, 256>(hip_bfloat16 const*, un" + }, + "top_kernels": [ + { + "kernel": "void mlx::core::rocm::qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int, int, bool)", + "class": "qmv", + "calls_per_token": 167.68, + "share_pct": 25.18 + }, + { + "kernel": "void mlx::core::rocm::gather_qmv_wide_kernel(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, unsigned int const*, unsigned int const*, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int, hip_bfloat16*, int, int, int, int, int, bool, bool, long)", + "class": "gather_qmv", + "calls_per_token": 119.06, + "share_pct": 18.9 + }, + { + "kernel": "void mlx::core::rocm::binary_vv(float const*, float const*, float*, unsigned int)", + "class": "binary", + "calls_per_token": 142.59, + "share_pct": 6.89 + }, + { + "kernel": "void mlx::core::rocm::binary_g(float const*, float const*, float*, long, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, mlx::core::rocm::hip_array, int)", + "class": "binary", + "calls_per_token": 178.31, + "share_pct": 5.77 + }, + { + "kernel": "void mlx::core::rocm::block_sort_kernel(hip_bfloat16 const*, unsigned int*, int, long, long, long, long)", + "class": "sort", + "calls_per_token": 39.69, + "share_pct": 3.73 + }, + { + "kernel": "void mlx::core::rocm::rms_norm_kernel(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, float, unsigned int, long)", + "class": "rms_norm", + "calls_per_token": 116.09, + "share_pct": 3.58 + }, + { + "kernel": "void mlx::core::rocm::copy_v(hip_bfloat16 const*, float*, unsigned int)", + "class": "copy", + "calls_per_token": 330.4, + "share_pct": 2.91 + }, + { + "kernel": "void mlx::core::rocm::CV2ISigmoidADV2IBroadcastACEV2IBroadcastCAFV2IMultiplyDEGV2IBroadcastFBHV2IBroadcastBFIV2OMultiplyGH_VV_V2V2_6142509188972423790_contiguous(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)", + "class": "compiled", + "calls_per_token": 79.38, + "share_pct": 2.9 + }, + { + "kernel": "void mlx::core::rocm::binary_vv(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)", + "class": "binary", + "calls_per_token": 154.78, + "share_pct": 2.56 + }, + { + "kernel": "void mlx::core::rocm::BV2ISigmoidACV2IBroadcastABDV2IBroadcastBAEV2OMultiplyCD_V_V2_6142509188972423790_contiguous(hip_bfloat16 const*, hip_bfloat16*, unsigned int)", + "class": "compiled", + "calls_per_token": 71.44, + "share_pct": 1.9 + } + ], + "class_share_pct": { + "qmv": 26.24, + "binary": 21.81, + "gather_qmv": 18.9, + "copy": 9.28, + "compiled": 4.8, + "gemm": 4.25, + "sort": 3.73, + "rms_norm": 3.58, + "unary": 1.98, + "ternary": 1.33, + "scan": 0.95, + "softmax": 0.8, + "reduce": 0.65, + "gather_scatter": 0.6, + "conv": 0.58, + "sdpa": 0.43, + "arg_reduce": 0.07, + "dequantize": 0.01 + }, + "role_share_pct": { + "unattributed": 37.33, + "ssm_step": 31.18, + "moe_expert_gemv": 18.9, + "add_rms_join": 3.57, + "ssm_gated_norm": 2.19, + "moe_activation": 1.94, + "ssm_silu": 1.9, + "ssm_conv": 1.38, + "moe_weighted_sum": 0.83, + "moe_gather_indices": 0.58, + "sampler_tail": 0.21, + "add_rms_join_post_attn": 0.0, + "rope_append": 0.0, + "paged_attention": 0.0 + }, + "role_dispatches_per_token": { + "unattributed": 669.9, + "ssm_step": 1678.5, + "ssm_conv": 107.2, + "ssm_silu": 71.4, + "ssm_gated_norm": 107.2, + "add_rms_join": 156.8, + "moe_gather_indices": 79.4, + "moe_expert_gemv": 119.1, + "moe_activation": 79.4, + "moe_weighted_sum": 79.4, + "sampler_tail": 9.9, + "add_rms_join_post_attn": 0.0, + "rope_append": 0.0, + "paged_attention": 0.0 + }, + "role_host_gap_ms_per_token": { + "ssm_step": 4.237, + "unattributed": 1.7191, + "moe_expert_gemv": 0.5928, + "add_rms_join": 0.3889, + "moe_activation": 0.3142, + "ssm_gated_norm": 0.2946, + "ssm_conv": 0.2648, + "moe_weighted_sum": 0.1893, + "moe_gather_indices": 0.1849, + "ssm_silu": 0.1699, + "sampler_tail": 0.0782, + "tail": 0.0009 + }, + "port_units": { + "2063": { + "port": "fused_add_rms_norm + fused_rope_qk_append", + "fallback_share_pct": 0.0, + "fallback_dispatches_per_token": 0.0, + "reached_default_share_pct": 0.0, + "reached_default_dispatches_per_token": 0.0, + "reached_optin_share_pct": 0.0, + "fallback_host_gap_ms_per_token": 0.0, + "fallback_host_gap_us_per_dispatch": 0.0, + "fallback_wall_share_pct": 0.0, + "ceiling_gpu_share": 1.0, + "ceiling_wall": 1.0, + "plain_ceiling_wall_est": 1.0, + "reached_default_host_gap_ms_per_token": 0.0, + "reached_default_host_gap_us_per_dispatch": 0.0, + "reached_default_wall_share_pct": 0.0, + "reached_default_ceiling_gpu_share": 1.0, + "reached_default_ceiling_wall": 1.0, + "reached_default_plain_ceiling_wall_est": 1.0, + "note": "granitemoehybrid never calls fused_add_rms_norm or forward_fused_rope_append" + }, + "2064": { + "port": "samplers (gumbel_max_sample, rejection_sample)", + "fallback_share_pct": 0.21, + "fallback_dispatches_per_token": 9.9, + "reached_default_share_pct": 0.0, + "reached_default_dispatches_per_token": 0.0, + "reached_optin_share_pct": 0.0, + "fallback_host_gap_ms_per_token": 0.0782, + "fallback_host_gap_us_per_dispatch": 7.89, + "fallback_wall_share_pct": 0.52, + "ceiling_gpu_share": 1.0, + "ceiling_wall": 1.01, + "plain_ceiling_wall_est": 1.0, + "reached_default_host_gap_ms_per_token": 0.0, + "reached_default_host_gap_us_per_dispatch": 0.0, + "reached_default_wall_share_pct": 0.0, + "reached_default_ceiling_gpu_share": 1.0, + "reached_default_ceiling_wall": 1.0, + "reached_default_plain_ceiling_wall_est": 1.0, + "note": "greedy argmax dispatches neither sampler kernel" + }, + "2065": { + "port": "fused MoE decode (moe_gateup, moe_down)", + "fallback_share_pct": 22.25, + "fallback_dispatches_per_token": 357.2, + "reached_default_share_pct": 0.0, + "reached_default_dispatches_per_token": 0.0, + "reached_optin_share_pct": 0.0, + "fallback_host_gap_ms_per_token": 1.2812, + "fallback_host_gap_us_per_dispatch": 3.59, + "fallback_wall_share_pct": 19.17, + "ceiling_gpu_share": 1.29, + "ceiling_wall": 1.24, + "plain_ceiling_wall_est": 1.25, + "reached_default_host_gap_ms_per_token": 0.0, + "reached_default_host_gap_us_per_dispatch": 0.0, + "reached_default_wall_share_pct": 0.0, + "reached_default_ceiling_gpu_share": 1.0, + "reached_default_ceiling_wall": 1.0, + "reached_default_plain_ceiling_wall_est": 1.0, + "note": "granitemoehybrid does not call forward_fused_kernel" + }, + "2067": { + "port": "SSM update (ssm_update_kernel)", + "fallback_share_pct": 31.18, + "fallback_dispatches_per_token": 1678.5, + "reached_default_share_pct": 31.18, + "reached_default_dispatches_per_token": 1678.5, + "reached_optin_share_pct": 31.18, + "fallback_host_gap_ms_per_token": 4.237, + "fallback_host_gap_us_per_dispatch": 2.52, + "fallback_wall_share_pct": 39.48, + "ceiling_gpu_share": 1.45, + "ceiling_wall": 1.65, + "plain_ceiling_wall_est": 1.6, + "reached_default_host_gap_ms_per_token": 4.237, + "reached_default_host_gap_us_per_dispatch": 2.52, + "reached_default_wall_share_pct": 39.48, + "reached_default_ceiling_gpu_share": 1.45, + "reached_default_ceiling_wall": 1.65, + "reached_default_plain_ceiling_wall_est": 1.6, + "note": "ssm_kernel_available() gates the decode step" + }, + "2068": { + "port": "paged attention (v1, v2, merge)", + "fallback_share_pct": 0.0, + "fallback_dispatches_per_token": 0.0, + "reached_default_share_pct": 0.0, + "reached_default_dispatches_per_token": 0.0, + "reached_optin_share_pct": 0.0, + "fallback_host_gap_ms_per_token": 0.0, + "fallback_host_gap_us_per_dispatch": 0.0, + "fallback_wall_share_pct": 0.0, + "ceiling_gpu_share": 1.0, + "ceiling_wall": 1.0, + "plain_ceiling_wall_est": 1.0, + "reached_default_host_gap_ms_per_token": 0.0, + "reached_default_host_gap_us_per_dispatch": 0.0, + "reached_default_wall_share_pct": 0.0, + "reached_default_ceiling_gpu_share": 1.0, + "reached_default_ceiling_wall": 1.0, + "reached_default_plain_ceiling_wall_est": 1.0, + "note": "the bench decodes into a dense KVCache; the paged path is not taken" + } + } +} diff --git a/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/guard.log b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/guard.log new file mode 100644 index 000000000..e828041b0 --- /dev/null +++ b/benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/guard.log @@ -0,0 +1,74 @@ +== granite-4.0-h-tiny-4bit_greedy plain +2026-10-08T20:31:29+0900 waiting for guard lock /tmp/mlxcel-rocm-gpu-guard.lock +2026-10-08T21:16:25+0900 acquired guard lock /tmp/mlxcel-rocm-gpu-guard.lock after 2696s +2026-10-08T21:16:25+0900 waiting for 90s of idle GPU and no compiler +2026-10-08T21:17:21+0900 idle streak reset after 38s: gpu=[] compilers=[2996121:cargo] +2026-10-08T21:20:49+0900 idle for 90s: kfd proc empty and no compiler in every 1 Hz sample +2026-10-08T21:20:49+0900 attempt 1/5: start: target/release/mlxcel-bench-decode -m models/mlx/granite-4.0-h-tiny-4bit -p profile -n 128 --warmup-tokens 20 --ignore-eos --prompt-tokens 512 --temperature 0 --top-p 1.0 +2026-10-08T21:20:49+0900 sample 1: clean gpu=[] +2026-10-08T21:20:50+0900 sample 2: clean gpu=[3156047:mlxcel-bench-de] +2026-10-08T21:20:52+0900 sample 3: clean gpu=[3156047:mlxcel-bench-de] +2026-10-08T21:20:53+0900 sample 4: clean gpu=[3156047:mlxcel-bench-de] +2026-10-08T21:20:55+0900 sample 5: clean gpu=[3156047:mlxcel-bench-de] +2026-10-08T21:20:56+0900 monitor: 5 samples +2026-10-08T21:20:56+0900 attempt 1: CLEAN, exit 0 +== granite-4.0-h-tiny-4bit_greedy profiled +2026-10-08T21:20:56+0900 waiting for guard lock /tmp/mlxcel-rocm-gpu-guard.lock +2026-10-08T21:25:35+0900 acquired guard lock /tmp/mlxcel-rocm-gpu-guard.lock after 279s +2026-10-08T21:25:35+0900 waiting for 90s of idle GPU and no compiler +2026-10-08T21:27:27+0900 idle streak reset after 76s: gpu=[] compilers=[3403424:cargo,3404312:hipcc,3404314:hipcc,3404319:hipcc,3404323:hipcc,3404328:hipcc,3404335:clang++,3404336:clang++,3404337:clang++,3404338:clang++,3404340:clang++,3404341:clang-23,3404342:clang-23,3404343:clang-23,3404344:clang-23,3404345:clang-23] +2026-10-08T21:32:01+0900 idle for 90s: kfd proc empty and no compiler in every 1 Hz sample +2026-10-08T21:32:01+0900 attempt 1/5: start: /usr/bin/rocprofv3 --kernel-trace --hip-graph-trace --stats -f csv -d /granite-4.0-h-tiny-4bit_greedy -o granite-4.0-h-tiny-4bit_greedy -- target/release/mlxcel-bench-decode -m models/mlx/granite-4.0-h-tiny-4bit -p profile -n 128 --warmup-tokens 20 --ignore-eos --prompt-tokens 512 --temperature 0 --top-p 1.0 +2026-10-08T21:32:01+0900 sample 1: clean gpu=[] +2026-10-08T21:32:03+0900 sample 2: clean gpu=[3572925:mlxcel-bench-de] +2026-10-08T21:32:04+0900 sample 3: clean gpu=[3572925:mlxcel-bench-de] +2026-10-08T21:32:06+0900 sample 4: clean gpu=[3572925:mlxcel-bench-de] +2026-10-08T21:32:08+0900 sample 5: clean gpu=[3572925:mlxcel-bench-de] +2026-10-08T21:32:09+0900 monitor: 5 samples +2026-10-08T21:32:09+0900 attempt 1: CLEAN, exit 0 +== NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy plain +2026-10-08T21:32:31+0900 waiting for guard lock /tmp/mlxcel-rocm-gpu-guard.lock +2026-10-08T22:11:29+0900 acquired guard lock /tmp/mlxcel-rocm-gpu-guard.lock after 2338s +2026-10-08T22:11:29+0900 waiting for 90s of idle GPU and no compiler +2026-10-08T22:13:04+0900 idle streak reset after 65s: gpu=[] compilers=[906429:cargo] +2026-10-08T22:18:37+0900 idle for 90s: kfd proc empty and no compiler in every 1 Hz sample +2026-10-08T22:18:37+0900 attempt 1/5: start: target/release/mlxcel-bench-decode -m models/mlx/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit -p profile -n 128 --warmup-tokens 20 --ignore-eos --prompt-tokens 512 --temperature 0 --top-p 1.0 +2026-10-08T22:18:37+0900 sample 1: clean gpu=[] +2026-10-08T22:18:39+0900 sample 2: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:40+0900 sample 3: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:42+0900 sample 4: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:43+0900 sample 5: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:45+0900 sample 6: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:46+0900 sample 7: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:48+0900 sample 8: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:49+0900 sample 9: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:50+0900 sample 10: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:52+0900 sample 11: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:53+0900 sample 12: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:55+0900 sample 13: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:56+0900 sample 14: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:58+0900 sample 15: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:18:59+0900 sample 16: clean gpu=[1129968:mlxcel-bench-de] +2026-10-08T22:19:00+0900 monitor: 16 samples +2026-10-08T22:19:00+0900 attempt 1: CLEAN, exit 0 +== NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy profiled +2026-10-08T22:19:00+0900 waiting for guard lock /tmp/mlxcel-rocm-gpu-guard.lock +2026-10-08T23:11:24+0900 acquired guard lock /tmp/mlxcel-rocm-gpu-guard.lock after 3144s +2026-10-08T23:11:24+0900 waiting for 90s of idle GPU and no compiler +2026-10-08T23:13:36+0900 idle for 90s: kfd proc empty and no compiler in every 1 Hz sample +2026-10-08T23:13:36+0900 attempt 1/5: start: /usr/bin/rocprofv3 --kernel-trace --hip-graph-trace --stats -f csv -d /NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy -o NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy -- target/release/mlxcel-bench-decode -m models/mlx/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit -p profile -n 128 --warmup-tokens 20 --ignore-eos --prompt-tokens 512 --temperature 0 --top-p 1.0 +2026-10-08T23:13:37+0900 sample 1: clean gpu=[] +2026-10-08T23:13:38+0900 sample 2: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:39+0900 sample 3: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:41+0900 sample 4: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:42+0900 sample 5: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:44+0900 sample 6: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:45+0900 sample 7: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:47+0900 sample 8: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:48+0900 sample 9: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:50+0900 sample 10: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:52+0900 sample 11: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:53+0900 sample 12: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:55+0900 sample 13: clean gpu=[3228542:mlxcel-bench-de] +2026-10-08T23:13:56+0900 monitor: 13 samples +2026-10-08T23:13:56+0900 attempt 1: CLEAN, exit 0 diff --git a/docs/benchmark_results/rocm-decode-profile-gfx1151-2026-09-30.md b/docs/benchmark_results/rocm-decode-profile-gfx1151-2026-09-30.md index a273745fb..64343c86e 100644 --- a/docs/benchmark_results/rocm-decode-profile-gfx1151-2026-09-30.md +++ b/docs/benchmark_results/rocm-decode-profile-gfx1151-2026-09-30.md @@ -126,6 +126,19 @@ A share is an upper bound on what a port saves, since the ported kernel costs so - **#2063** is zero with shipped settings on every backend. Turned on, the most it could reach here is Llama's post-attention join, 0.83% of decode (0.89% in the top-p run). - **#2068** has nothing to act on in single-stream decode. Its value is the batched paged serving path, which this profile does not measure, and the 36 paged-attention test skips on ROCm. +## Note, 2026-10-09: the #2067 shares understated the gain (issue #2148) + +The shares above are of decode GPU time, so the ceiling they imply, `1 / (1 - share)`, leaves out the host launch gaps between dispatches. For #2067 that bound was 1.42x on granite and 1.25x on Nemotron-H, yet the port measured 1.46x and 1.45x (PR #2099, [rocm-ssm-update-kernel-gfx1151-2026-10-04.md](rocm-ssm-update-kernel-gfx1151-2026-10-04.md)): it removes more than half of each model's dispatches, and the gaps in front of them go with them. `scripts/rocm_decode_profile.py` now charges each idle stretch of the decode window to the dispatch that ends it and reports, per port unit, its share of decode wall time and the ceiling that implies (`ceiling_wall`; `plain_ceiling_wall_est` scales the traced gaps to the run without the profiler, see `docs/benchmarks.md`). + +Re-profiled at `6dbfe7d7` with `MLXCEL_SSM_KERNEL=0` (the graph path this page measured), same shape and guard, results in `benchmarks/rocm_profiles/gfx1151_6dbfe7d7_ssm-graph/`. The measured speedup is the median of three alternating runs per arm of the same binary, kernel off and on, in one guard session per model (`ab_decode.txt`, `ab_guard.log`): + +| Model | #2067 GPU share | Ceiling, GPU share | #2067 host gap ms/token (us/dispatch, profiled) | Ceiling, wall (profiled) | Ceiling, wall (plain estimate) | Decode tok/s off / on (medians) | Measured | +|---|---:|---:|---:|---:|---:|---:|---:| +| `granite-4.0-h-tiny-4bit` | 31.18% | 1.45x | 4.24 (2.5) | 1.65x | 1.60x | 59.88 / 89.40 | 1.49x | +| `NVIDIA-Nemotron-3-Nano-30B-A3B-4bit` | 20.70% | 1.26x | 6.73 (5.9) | 1.72x | 1.56x | 52.29 / 75.26 | 1.44x | + +Spread: kernel on, 89.33 to 89.43 and 75.04 to 75.37 tok/s; kernel off, 59.88 to 60.85 on granite after a first run at 52.18, and 52.06 to 52.66 on Nemotron. The plain-run ceilings sit above the measured gains, as an upper bound must (the ported kernel and its one dispatch per layer still cost time); the GPU-share ceilings sit below them. Rank dispatch-heavy ports by `plain_ceiling_wall_est`, not by GPU share. + ## Ranked port order By share of decode GPU time reached with shipped settings, weighed by how much of it a kernel can recover: diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 053bfd084..551d328f4 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -268,7 +268,22 @@ to a temporary directory (`--trace-dir` to keep them) because they run to hundreds of MB. Read shares rather than absolute times from the profiled run: the tracer slows decode, and the summary records by how much against the plain run. `--temperature` and `--top-p` exist so a profile can see the sampler; the -greedy default never dispatches it. The published profile and how each kernel +greedy default never dispatches it. + +A port unit's share of decode GPU time implies a speedup ceiling of +`1 / (1 - share)` (`ceiling_gpu_share` in the summary), but that share ignores +the host launch gaps between dispatches, so it underrates a port that replaces +many small dispatches with one kernel (issue #2148). The summary therefore also +charges each idle stretch of the decode window to the dispatch that ends it +(`scripts/rocm_decode_gaps.py`): per role `role_host_gap_ms_per_token`, and per +port unit its host gap per token and per dispatch, its share of decode wall +time (kernels plus the gaps in front of them) and `ceiling_wall`, the speedup +bound that share implies. With a plain run the gaps are scaled to the plain +run's host gap (the tracer inflates host gaps, not kernel time) and divided by +its wall time per token: `plain_ceiling_wall_est`. That is the bound to compare +a port's measured speedup against; `report` prints it beside the GPU-share +ceiling. The same fields exist for the roles reached with shipped defaults +(`reached_default_*`). The published profile and how each kernel name was attributed to a port unit are in [rocm-decode-profile-gfx1151-2026-09-30.md](benchmark_results/rocm-decode-profile-gfx1151-2026-09-30.md). diff --git a/scripts/rocm_decode_gaps.py b/scripts/rocm_decode_gaps.py new file mode 100644 index 000000000..84638f046 --- /dev/null +++ b/scripts/rocm_decode_gaps.py @@ -0,0 +1,132 @@ +# Copyright 2025-2026 Lablup Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Host-gap accounting per role and port unit for the ROCm decode profile. + +Issue #2148. A port unit's share of decode GPU time implies a speedup ceiling +of ``1 / (1 - share)``, but a port that replaces many small dispatches also +removes the host time spent launching them, which that share leaves out. Here +every idle stretch of the decode window is charged to the dispatch that ends +it, so each role (and each port unit, a set of roles) owns its kernels' time +plus the launch gaps in front of them, and its share of the wall time gives a +ceiling that counts both. + +``scripts/rocm_decode_profile.py`` imports this module; its tests live in +``tests/test_rocm_decode_profile.py``. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Iterable, Mapping, Protocol, Sequence + + +class Interval(Protocol): + start: int + end: int + + +def attribute_gaps(decode: Sequence[Interval], roles: Sequence[str | None], + lo: int, hi: int) -> dict[str, int]: + """Idle nanoseconds of ``[lo, hi]`` per role of the dispatch that ends them. + + ``decode`` must be in start order and start inside the window (the decode + cut guarantees both). Walking it, ``frontier`` is the latest end seen so + far (at least ``lo``); a dispatch starting after the frontier ends a gap of + ``start - frontier``, charged to its role (``"unattributed"`` for None). A + dispatch that overlaps the frontier ends no gap. The idle stretch from the + last end to ``hi`` is ``"tail"``. Ends are clipped to ``hi`` as + ``busy_ns`` clips them, so the values sum to ``(hi - lo) - busy_ns(decode, + lo, hi)``. + """ + if len(decode) != len(roles): + raise ValueError(f"{len(decode)} dispatches but {len(roles)} roles") + gaps: dict[str, int] = defaultdict(int) + frontier = lo + for d, r in zip(decode, roles): + gaps[r or "unattributed"] += max(0, min(d.start, hi) - frontier) + frontier = max(frontier, min(d.end, hi)) + gaps["tail"] += max(0, hi - frontier) + return dict(gaps) + + +def ceiling(fraction: float | None) -> float | None: + """``1 / (1 - fraction)`` to 2 decimals; None when undefined (a share of 100%).""" + if fraction is None or fraction >= 1.0: + return None + return round(1.0 / (1.0 - max(fraction, 0.0)), 2) + + +def gap_scale(plain_host_gap_ms_per_token: float | None, + host_gap_ms_per_token: float | None) -> float | None: + """How much of the traced host gap a run without the profiler keeps. + + The tracer adds host time per dispatch but barely changes kernel durations, + so each role's traced gap is scaled by the plain run's estimated gap over + the traced one. None without a plain run; never negative. + """ + if plain_host_gap_ms_per_token is None or host_gap_ms_per_token is None: + return None + if host_gap_ms_per_token <= 0: + return 0.0 + return max(plain_host_gap_ms_per_token, 0.0) / host_gap_ms_per_token + + +def wall_fields(prefix: str, roles: Iterable[str], role_ns: Mapping[str, int], + role_gap_ns: Mapping[str, int], role_calls: Mapping[str, int], *, + gpu_sum: int, wall: int, tokens: int, scale: float | None, + plain_wall_ms_per_token: float | None) -> dict[str, float | None]: + """Host gap, wall share and both ceilings of the roles a port would replace. + + ``prefix`` names the role set (``fallback`` or ``reached_default``). + ``ceiling_gpu_share`` is the old GPU-time bound; ``ceiling_wall`` adds the + roles' launch gaps and divides by the window's wall time; + ``plain_ceiling_wall_est`` does the same against the run without the + profiler, the bound to compare a port's measured speedup against. Every key + takes the prefix except the fallback set's three ceilings, which are + ``ceiling_gpu_share``, ``ceiling_wall`` and ``plain_ceiling_wall_est``. + """ + roles = tuple(roles) + kernel_ns = sum(role_ns.get(r, 0) for r in roles) + gap_ns = sum(role_gap_ns.get(r, 0) for r in roles) + calls = sum(role_calls.get(r, 0) for r in roles) + wall_frac = (kernel_ns + gap_ns) / wall if wall > 0 else None + plain_frac = None + if scale is not None and plain_wall_ms_per_token and tokens: + unit_ms_per_token = (kernel_ns + gap_ns * scale) / 1e6 / tokens + plain_frac = unit_ms_per_token / plain_wall_ms_per_token + ceil = "" if prefix == "fallback" else f"{prefix}_" + return { + f"{prefix}_host_gap_ms_per_token": round(gap_ns / 1e6 / tokens, 4) if tokens else None, + f"{prefix}_host_gap_us_per_dispatch": round(gap_ns / 1e3 / calls, 2) if calls else 0.0, + f"{prefix}_wall_share_pct": round(100.0 * wall_frac, 2) if wall_frac is not None else None, + f"{ceil}ceiling_gpu_share": ceiling(kernel_ns / gpu_sum) if gpu_sum > 0 else None, + f"{ceil}ceiling_wall": ceiling(wall_frac), + f"{ceil}plain_ceiling_wall_est": ceiling(plain_frac), + } + + +def ceiling_cell(unit: Mapping[str, object]) -> str: + """``wall (gpu share)`` for the report; the plain estimate when there is one.""" + wall = unit.get("plain_ceiling_wall_est") + if wall is None: + wall = unit.get("ceiling_wall") + gpu = unit.get("ceiling_gpu_share") + if wall is None and gpu is None: + return "-" + return f"{_fmt(wall)} ({_fmt(gpu)})" + + +def _fmt(v: object) -> str: + return "-" if v is None else f"{v:.2f}" diff --git a/scripts/rocm_decode_profile.py b/scripts/rocm_decode_profile.py index 2ce063751..67bec13c5 100644 --- a/scripts/rocm_decode_profile.py +++ b/scripts/rocm_decode_profile.py @@ -25,6 +25,12 @@ * ``_summary.json``: decode GPU time and host gap per token, tok/s with and without the profiler, share per port unit, and the checks below. +Per port unit the summary also charges each idle stretch of the decode window +to the dispatch that ends it (``rocm_decode_gaps.py``, issue #2148), so a unit +owns its launch gaps as well as its kernel time; ``ceiling_wall`` (and +``plain_ceiling_wall_est`` against the run without the profiler) is the speedup +bound that counts both, where ``ceiling_gpu_share`` counts kernel time only. + The measured decode is the dispatches that start between the bench's ``decode_start`` and ``measured_end`` marks (see ``src/bin/bench_decode/phase_marks.rs``). The prefill before it ends in a @@ -49,6 +55,8 @@ from collections import defaultdict from dataclasses import dataclass, field +from rocm_decode_gaps import attribute_gaps, ceiling_cell, gap_scale, wall_fields + # -------------------------------------------------------------------------- # Kernel classes. First match wins. Patterns run on the demangled kernel name # as rocprofv3 writes it; see the profile doc for how each was checked against @@ -434,6 +442,21 @@ def summarize(trace: pathlib.Path, log_path: pathlib.Path, plain_log: pathlib.Pa role_ns[r or "unattributed"] += d.dur role_calls[r or "unattributed"] += 1 class_ns[classify(d.name)] += d.dur + role_gap_ns = attribute_gaps(decode, roles, lo, hi) + plain = parse_log(plain_log.read_text(errors="replace")) if plain_log else None + host_gap_ms_per_token = round((wall - gpu_busy) / 1e6 / tokens, 4) if tokens else None + plain_wall_ms_per_token = round(1000.0 / plain.decode_tok_s, 4) \ + if plain and plain.decode_tok_s else None + # The tracer adds host time per dispatch but barely changes kernel + # durations, so the unprofiled run's wall time per token minus the traced + # GPU time is the better host-gap estimate for dispatch-heavy models. Same + # denominator as tok/s (generated tokens). + plain_host_gap_ms_per_token_est = \ + round(1000.0 / plain.decode_tok_s - gpu_busy / 1e6 / tokens, 4) \ + if plain_wall_ms_per_token is not None and tokens else None + gap_kw = {"gpu_sum": gpu_sum, "wall": wall, "tokens": tokens, + "scale": gap_scale(plain_host_gap_ms_per_token_est, host_gap_ms_per_token), + "plain_wall_ms_per_token": plain_wall_ms_per_token} def share(ns: int) -> float: return round(100.0 * ns / gpu_sum, 2) @@ -463,10 +486,12 @@ def share(ns: int) -> float: "reached_default_dispatches_per_token": round(sum(role_calls[r] for r in default_roles) / tokens, 1) if tokens else None, "reached_optin_share_pct": share(sum(role_ns[r] for r in optin_roles)), + **wall_fields("fallback", rs, role_ns, role_gap_ns, role_calls, **gap_kw), + **wall_fields("reached_default", default_roles, role_ns, role_gap_ns, role_calls, + **gap_kw), "note": note, } - plain = parse_log(plain_log.read_text(errors="replace")) if plain_log else None top = sorted(per_kernel.items(), key=lambda kv: -kv[1][1])[:10] summary = { "name": name, @@ -487,19 +512,15 @@ def share(ns: int) -> float: "decode_gpu_sum_ms": round(gpu_sum / 1e6, 3), "decode_gpu_busy_ms": round(gpu_busy / 1e6, 3), "gpu_ms_per_token": round(gpu_busy / 1e6 / tokens, 4) if tokens else None, - "host_gap_ms_per_token": round((wall - gpu_busy) / 1e6 / tokens, 4) if tokens else None, + "host_gap_ms_per_token": host_gap_ms_per_token, "host_gap_pct_of_wall": round(100.0 * (wall - gpu_busy) / wall, 2), - # The tracer adds host time per dispatch but barely changes kernel - # durations, so the unprofiled run's wall time per token minus the - # traced GPU time is the better host-gap estimate for dispatch-heavy - # models. Same denominator as tok/s (generated tokens). - "plain_wall_ms_per_token": round(1000.0 / plain.decode_tok_s, 4) - if plain and plain.decode_tok_s else None, - "plain_host_gap_ms_per_token_est": - round(1000.0 / plain.decode_tok_s - gpu_busy / 1e6 / tokens, 4) - if plain and plain.decode_tok_s and tokens else None, + "plain_wall_ms_per_token": plain_wall_ms_per_token, + "plain_host_gap_ms_per_token_est": plain_host_gap_ms_per_token_est, "checks": { "kernels_straddling_decode_start": straddling, + # The per-role host gaps must add up to the window's idle time. + "host_gap_attribution_residual_ns": + sum(role_gap_ns.values()) - (wall - gpu_busy), "idle_gap_before_first_decode_dispatch_us": round((decode[0].start - before[-1].end) / 1e3, 1) if before else None, "last_dispatch_before_decode": before[-1].name[:120] if before else None, @@ -513,6 +534,11 @@ def share(ns: int) -> float: "role_share_pct": {r: share(ns) for r, ns in sorted(role_ns.items(), key=lambda kv: -kv[1])}, "role_dispatches_per_token": {r: round(c / tokens, 1) for r, c in role_calls.items()} if tokens else {}, + # Idle time before each role's dispatches ("tail": after the last one). + "role_host_gap_ms_per_token": { + r: round(ns / 1e6 / tokens, 4) + for r, ns in sorted(role_gap_ns.items(), key=lambda kv: -kv[1]) + } if tokens else {}, "port_units": units, } (out_dir / f"{name}_summary.json").write_text(json.dumps(summary, indent=2) + "\n") @@ -561,6 +587,22 @@ def report(out_dir: pathlib.Path) -> str: p = s["port_units"][u] cells.append(f"{p['reached_default_share_pct']} ({p['fallback_share_pct']})") lines.append(f"| `{s['name']}` | " + " | ".join(cells) + " |") + # Speedup ceilings of each unit's fallback roles: wall time (kernels plus the + # launch gaps in front of them, against the plain run when there is one), + # then in brackets the GPU-time-share ceiling, which ignores the gaps. + lines += ["", "| Run: ceiling wall (GPU share) | " + + " | ".join(f"#{u}" for u in UNIT_ROLES) + " |", + "|---|" + "---:|" * len(UNIT_ROLES)] + for s in rows: + units = s["port_units"] + if any(units[u].get("plain_ceiling_wall_est") is not None for u in UNIT_ROLES): + basis = "plain" + elif any(units[u].get("ceiling_wall") is not None for u in UNIT_ROLES): + basis = "profiled" + else: + basis = "summary predates #2148" + cells = [ceiling_cell(units[u]) for u in UNIT_ROLES] + lines.append(f"| `{s['name']}` ({basis}) | " + " | ".join(cells) + " |") for s in rows: lines += ["", f"Top kernels, `{s['name']}` (share of decode GPU time, calls per token):", ""] for k in s["top_kernels"]: diff --git a/tests/test_rocm_decode_profile.py b/tests/test_rocm_decode_profile.py index e780f650a..009b26dd0 100644 --- a/tests/test_rocm_decode_profile.py +++ b/tests/test_rocm_decode_profile.py @@ -15,7 +15,8 @@ """Tests for the ROCm per-kernel decode profile tooling (issue #2061). Covers scripts/rocm_decode_profile.py (the decode cut, the busy-time union, -the port-unit attribution) on synthetic traces, and scripts/rocm_gpu_guard.sh +the port-unit attribution) and scripts/rocm_decode_gaps.py (host gaps per +role and the wall-time ceilings, issue #2148) on synthetic traces, and scripts/rocm_gpu_guard.sh against a fake KFD process directory. No GPU is needed. Run with: @@ -35,6 +36,7 @@ GUARD = ROOT / "scripts" / "rocm_gpu_guard.sh" sys.path.insert(0, str(ROOT / "scripts")) +import rocm_decode_gaps as gaps # noqa: E402 import rocm_decode_profile as rdp # noqa: E402 # A compiler pattern nothing on the host matches, so the guard's compiler check @@ -437,13 +439,105 @@ def test_summarize_cuts_the_decode_window_by_the_phase_marks(self): "[phase] measured_end monotonic_ns=4 boottime_ns=800000\n" " Prompt tokens: 512\n Generated tokens: 2\n" " Prefill: 1.00 ms (512.00 tok/s)\n Decode: 0.20 ms (10000.00 tok/s)\n") - s = rdp.summarize(trace, log, None, None, "t", tmp) + plain = tmp / "plain.log" + plain.write_text(" Generated tokens: 2\n" + " Decode: 0.20 ms (8000.00 tok/s)\n") + s = rdp.summarize(trace, log, plain, None, "t", tmp) self.assertEqual(s["clock"], "boottime") self.assertEqual(s["decode_dispatches"], 2) self.assertEqual(s["decode_gpu_busy_ms"], 0.05) self.assertEqual(s["checks"]["kernels_straddling_decode_start"], 0) self.assertEqual(s["checks"]["idle_gap_before_first_decode_dispatch_us"], 110.0) self.assertTrue((tmp / "t_decode_kernels.csv").exists()) + # #2148: the window's 150 us of idle time, per role, adds up. + self.assertEqual(s["checks"]["host_gap_attribution_residual_ns"], 0) + self.assertEqual(s["role_host_gap_ms_per_token"], + {"unattributed": 0.055, "tail": 0.02}) + self.assertEqual(s["plain_wall_ms_per_token"], 0.125) + self.assertEqual(s["plain_host_gap_ms_per_token_est"], 0.1) + for unit in rdp.UNIT_ROLES: + p = s["port_units"][unit] + # No dispatch of any unit's roles: no gap, nothing to gain. + for prefix in ("fallback", "reached_default"): + self.assertEqual(p[f"{prefix}_host_gap_ms_per_token"], 0.0) + self.assertEqual(p[f"{prefix}_host_gap_us_per_dispatch"], 0.0) + self.assertEqual(p[f"{prefix}_wall_share_pct"], 0.0) + for key in ("ceiling_gpu_share", "ceiling_wall", "plain_ceiling_wall_est", + "reached_default_ceiling_gpu_share", "reached_default_ceiling_wall", + "reached_default_plain_ceiling_wall_est"): + self.assertEqual(p[key], 1.0, (unit, key)) + text = rdp.report(tmp) + self.assertIn("| Run: ceiling wall (GPU share) |", text) + self.assertIn("| `t` (plain) | 1.00 (1.00) |", text) + + +def disp(start, end, name="k"): + return rdp.Dispatch(name, start, end) + + +class GapTests(unittest.TestCase): + def test_gaps_go_to_the_dispatch_that_ends_them_and_sum_to_the_idle_time(self): + # A leading gap, an overlap, a dispatch inside another, an idle tail. + d = [disp(10, 20), disp(15, 30), disp(40, 50), disp(45, 48), disp(60, 70)] + roles = ["ssm_step", "rope_append", None, "ssm_step", "rope_append"] + gaps = rdp.attribute_gaps(d, roles, 0, 100) + self.assertEqual(gaps, {"ssm_step": 10, "rope_append": 10, "unattributed": 10, + "tail": 30}) + self.assertEqual(sum(gaps.values()), 100 - rdp.busy_ns(d, 0, 100)) + + def test_a_dispatch_running_past_the_window_leaves_no_tail(self): + d = [disp(5, 10), disp(95, 130)] + gaps = rdp.attribute_gaps(d, ["sampler_tail", "ssm_step"], 0, 100) + self.assertEqual(gaps, {"sampler_tail": 5, "ssm_step": 85, "tail": 0}) + self.assertEqual(sum(gaps.values()), 100 - rdp.busy_ns(d, 0, 100)) + + def test_role_count_must_match(self): + with self.assertRaises(ValueError): + rdp.attribute_gaps([disp(1, 2)], [], 0, 10) + + def test_wall_ceiling_counts_the_launch_gaps(self): + # 3 ms of a unit's kernels and 2 ms of gaps in front of them, in a 20 ms + # window with 10 ms of kernels; the plain run keeps half of each gap and + # takes 12 ms. + f = gaps.wall_fields("fallback", ("ssm_step",), {"ssm_step": 3_000_000}, + {"ssm_step": 2_000_000}, {"ssm_step": 4}, + gpu_sum=10_000_000, wall=20_000_000, tokens=1, scale=0.5, + plain_wall_ms_per_token=12.0) + self.assertEqual(f, { + "fallback_host_gap_ms_per_token": 2.0, + "fallback_host_gap_us_per_dispatch": 500.0, + "fallback_wall_share_pct": 25.0, + "ceiling_gpu_share": 1.43, # 1 / (1 - 0.30) + "ceiling_wall": 1.33, # 1 / (1 - 0.25) + "plain_ceiling_wall_est": 1.5, # 1 / (1 - (3 + 1) / 12) + }) + r = gaps.wall_fields("reached_default", (), {}, {}, {}, gpu_sum=1, wall=1, + tokens=1, scale=None, plain_wall_ms_per_token=None) + self.assertEqual(r["reached_default_ceiling_wall"], 1.0) + self.assertIsNone(r["reached_default_plain_ceiling_wall_est"]) + + def test_a_full_share_and_zero_tokens_have_no_ceiling_or_per_token_figure(self): + f = gaps.wall_fields("fallback", ("ssm_step",), {"ssm_step": 10}, {"ssm_step": 0}, + {"ssm_step": 1}, gpu_sum=10, wall=10, tokens=0, scale=1.0, + plain_wall_ms_per_token=1.0) + self.assertIsNone(f["ceiling_gpu_share"]) + self.assertIsNone(f["ceiling_wall"]) + self.assertIsNone(f["fallback_host_gap_ms_per_token"]) + self.assertIsNone(f["plain_ceiling_wall_est"]) + self.assertIsNone(gaps.ceiling(None)) + + def test_gap_scale(self): + self.assertIsNone(gaps.gap_scale(None, 4.0)) + self.assertEqual(gaps.gap_scale(2.0, 4.0), 0.5) + self.assertEqual(gaps.gap_scale(-1.0, 4.0), 0.0) + self.assertEqual(gaps.gap_scale(1.0, 0.0), 0.0) + + def test_report_cell_prefers_the_plain_estimate(self): + self.assertEqual(gaps.ceiling_cell({"ceiling_wall": 1.6, "plain_ceiling_wall_est": 1.5, + "ceiling_gpu_share": 1.42}), "1.50 (1.42)") + self.assertEqual(gaps.ceiling_cell({"ceiling_wall": 1.6, "plain_ceiling_wall_est": None, + "ceiling_gpu_share": None}), "1.60 (-)") + self.assertEqual(gaps.ceiling_cell({"fallback_share_pct": 3.0}), "-") if __name__ == "__main__":