Skip to content

perf(rocm): add host-gap accounting per port unit to the decode profile #2238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
inureyes merged 1 commit into from
Oct 8, 2026
Merged

perf(rocm): add host-gap accounting per port unit to the decode profile #2238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions ..._profiles/gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_bench.log
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
[mlx-rocm] bound HIP device 0: gfx1151 (AMD Radeon 8060S Graphics) cus=20 warp=32 lds=64KB
[Memory] cache limit: 2.15 GB
[Load] wall: 0.285 s MLX peak: 0.00 GB
[Memory] after load: active 0.00 GB, cache 0.00 GB, active+cache 0.00 GB, phase peak 0.00 GB
[long-prompt] target=512 tokens -> using 512 tokens (max_context=None, reserved 128 for generation)
[phase] warmup_start monotonic_ns=873860871948324 boottime_ns=873860871948374
[mlx-rocm] gfx1151 native_wmma=1 cus=20 warp=32 low_cu=0 host_WARP_SIZE=32
[hipBLASLt caps] device 0 (gfx1151): bf16=1 fp8_e4m3=0 fp8_e5m2=0 int8=1
[Memory] after warmup (prefill + warmup tokens): active 18.01 GB, cache 0.12 GB, active+cache 18.13 GB, phase peak 18.17 GB
[long-prompt] target=512 tokens -> using 512 tokens (max_context=None, reserved 128 for generation)
[phase] measured_start monotonic_ns=873873042710977 boottime_ns=873873042710997
[phase] decode_start monotonic_ns=873874739164814 boottime_ns=873874739164834
[phase] measured_end monotonic_ns=873877675065738 boottime_ns=873877675065758
[Memory] after measured pass: active 18.01 GB, cache 0.17 GB, active+cache 18.18 GB, phase peak 18.66 GB
[Profile Results]
Prompt tokens: 512
Generated tokens: 128
Prefill: 1694.32 ms (302.19 tok/s)
Decode: 2935.90 ms (43.60 tok/s)
Decode path: engine
MLX peak memory: 18.66 GB
52 changes: 52 additions & 0 deletions .../gfx1151_6dbfe7d7_ssm-graph/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit_greedy_decode_kernels.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
kernel,class,roles,port_units,calls,calls_per_token,total_ns,avg_ns,share_pct
"void mlx::core::rocm::qmv_wide_kernel<hip_bfloat16, hip_bfloat16, 64, 4>(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int, int, bool)",qmv,,,14859,116.09,706264203,47531,39.613
"void mlx::core::rocm::gather_qmv_wide_kernel<hip_bfloat16, hip_bfloat16, 64>(hip_bfloat16 const*, unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, unsigned int const*, unsigned int const*, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, mlx::core::rocm::hip_array<long, 10>, int, hip_bfloat16*, int, int, int, int, int, bool, bool, long)",gather_qmv,moe_expert_gemv,2065,5842,45.64,467734830,80064,26.234
"void mlx::core::rocm::binary_vv<mlx::core::rocm::Add, float, float, unsigned int, 4>(float const*, float const*, float*, unsigned int)",binary,sampler_tail;ssm_step,2064;2067,11811,92.27,75280902,6374,4.222
"void mlx::core::rocm::binary_g<mlx::core::rocm::Multiply, float, float, long, 1>(float const*, float const*, float*, long, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, mlx::core::rocm::hip_array<long, 10>, int)",binary,ssm_step,2067,14605,114.1,61815449,4232,3.467
"void mlx::core::rocm::gemv_batched_inline<float, 16, 4>(float const*, float const*, float*, int, int, mlx::core::rocm::GemvBatchParams)",gemm,ssm_step,2067,5842,45.64,39780619,6809,2.231
"void mlx::core::rocm::block_sort_kernel<float, unsigned int, true, 32, 8>(float const*, unsigned int*, int, long, long, long, long)",sort,,,2921,22.82,38292952,13110,2.148
"void mlx::core::rocm::copy_v<hip_bfloat16, float, unsigned int, 4>(hip_bfloat16 const*, float*, unsigned int)",copy,sampler_tail;ssm_gated_norm;ssm_step,2064;2067,26416,206.38,35912853,1360,2.014
"void mlx::core::rocm::rms_norm_kernel<hip_bfloat16, 256, 4>(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, float, unsigned int, long)",rms_norm,add_rms_join;add_rms_join_post_attn,2063,6731,52.59,29870307,4438,1.675
"void mlx::core::rocm::gemv_single<hip_bfloat16, 16, 4>(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, int, int)",gemm,,,2921,22.82,28067669,9609,1.574
"void mlx::core::rocm::binary_vv<mlx::core::rocm::Add, hip_bfloat16, hip_bfloat16, unsigned int, 4>(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned int)",binary,add_rms_join;add_rms_join_post_attn;ssm_step,2063;2067,12446,97.23,23934143,1923,1.342
"void mlx::core::rocm::binary_vv<mlx::core::rocm::Multiply, float, float, unsigned int, 4>(float const*, float const*, float*, unsigned int)",binary,ssm_gated_norm;ssm_step,2067,11684,91.28,20571906,1761,1.154
"void mlx::core::rocm::copy_s<float, float, unsigned int, 4>(float const*, float*, unsigned int)",copy,ssm_step,2067,11684,91.28,16388515,1403,0.919
"void mlx::core::rocm::copy_g_byval<float, float, long>(float const*, float*, long, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, int)",copy,ssm_step,2067,8763,68.46,15696777,1791,0.880
Cijk_Alik_Bjlk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8,gemm,ssm_step,2067,2921,22.82,14268641,4885,0.800
"void mlx::core::rocm::unary_v<mlx::core::rocm::Exp, float, float, unsigned int, 4>(float const*, float*, unsigned int)",unary,ssm_step,2067,8763,68.46,14070768,1606,0.789
"void mlx::core::rocm::ternary_g<mlx::core::rocm::Select, float, long, 4>(bool const*, float const*, float const*, float*, long, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, mlx::core::rocm::hip_array<long, 10>, mlx::core::rocm::hip_array<long, 10>, int)",ternary,ssm_step,2067,5842,45.64,13905548,2380,0.780
"void mlx::core::rocm::arange_kernel<int>(int*, int, int, unsigned long)",copy,ssm_step,2067,11684,91.28,13423189,1149,0.753
"void mlx::core::rocm::copy_v<float, hip_bfloat16, unsigned int, 4>(float const*, hip_bfloat16*, unsigned int)",copy,moe_activation;sampler_tail;ssm_gated_norm;ssm_step,2064;2065;2067,8890,69.45,13265591,1492,0.744
"void mlx::core::rocm::CV2IAsTypeBDV2IBroadcastACEV2IBroadcastCAFV2IMaximumDEGV2OSquareF_VC_V2_2297668033614959926_contiguous<unsigned int, 16>(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",compiled,moe_activation,2065,5842,45.64,12821809,2195,0.719
"void mlx::core::rocm::copy_gg_byval<hip_bfloat16, hip_bfloat16, long>(hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, mlx::core::rocm::hip_array<long, 10>, int)",copy,ssm_conv,,7390,57.73,12506124,1692,0.701
"void mlx::core::rocm::BV2ISigmoidACV2IBroadcastABDV2IBroadcastBAEV2OMultiplyCD_V_V2_6142509188972423790_contiguous<unsigned int, 16>(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",compiled,ssm_silu,,2921,22.82,10142188,3472,0.569
"void mlx::core::rocm::kernel_sdpav_1pass<hip_bfloat16, false, 128>(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, hip_bfloat16 const*, mlx::core::rocm::AttnParams)",sdpa,,,762,5.95,9153734,12013,0.513
"void mlx::core::rocm::col_reduce_small<hip_bfloat16, hip_bfloat16, mlx::core::rocm::Sum, 4>(hip_bfloat16 const*, hip_bfloat16*, mlx::core::rocm::ColReduceArgs, unsigned long)",reduce,,,2921,22.82,8387734,2872,0.470
"void mlx::core::rocm::binary_ss<mlx::core::rocm::GreaterEqual, int, bool, unsigned int, 4>(int const*, int const*, bool*, unsigned int)",binary,ssm_step,2067,5842,45.64,7838668,1342,0.440
Cijk_Ailk_Bljk_SB_MT32x32x8_SN_1LDSB0_AMAS0_BL1_BS1_EPS0_GLVWA1_GLVWB1_GRVW1_GSU1_GSUASB_ISA1151_IU1_K1_KLA_LBSPPA0_LBSPPB0_LPA0_LPB0_LRVW1_MIAV0_MMFGLC_NLCA1_NLCB1_PGR0_PLR1_SIA1_SS0_SU32_SUS256_SVW4_TT2_2_TLDS0_UMLDSA0_UMLDSB0_USFGROn1_VAW1_VSn1_VW1_VWB1_WSGRA0_WSGRB0_WS64_WG16_16_1_WGM8,gemm,ssm_step,2067,2921,22.82,7639224,2615,0.428
"void mlx::core::rocm::binary_vv<mlx::core::rocm::LogAddExp, float, float, unsigned int, 4>(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,2921,22.82,7027372,2406,0.394
"void mlx::core::(anonymous namespace)::depthwise_conv1d_kernel<hip_bfloat16>(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, mlx::core::ConvParams<1>)",conv,ssm_conv,,2921,22.82,6733185,2305,0.378
"void mlx::core::rocm::binary_vs<mlx::core::rocm::Minimum, float, float, unsigned int, 4>(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,2921,22.82,6361108,2178,0.357
"void mlx::core::rocm::rms_norm_kernel<float, 256, 4>(float const*, float const*, float*, float, unsigned int, long)",rms_norm,ssm_gated_norm,,2921,22.82,5926723,2029,0.332
"void mlx::core::rocm::Bf4ISigmoidACf4IBroadcastABDf4IBroadcastBAEf4OMultiplyCD_V_f4_6142509188972423790_contiguous<unsigned int, 8>(float const*, float*, unsigned int)",compiled,ssm_silu,,2921,22.82,5595526,1916,0.314
"void mlx::core::rocm::unary_v<mlx::core::rocm::Negative, float, float, unsigned int, 4>(float const*, float*, unsigned int)",unary,ssm_step,2067,2921,22.82,5592707,1915,0.314
"void mlx::core::rocm::contiguous_scan<float, float, mlx::core::rocm::Sum, 4, true, false>(float const*, float*, int)",scan,ssm_step,2067,2921,22.82,5399681,1849,0.303
"void mlx::core::rocm::arange_kernel<unsigned int>(unsigned int*, unsigned int, unsigned int, unsigned long)",copy,moe_gather_indices,2065,5842,45.64,5211505,892,0.292
"void mlx::core::rocm::strided_scan<float, float, mlx::core::rocm::Sum, 4, 32, 32, true, false>(float const*, float*, int, long, long)",scan,ssm_step,2067,2921,22.82,5119157,1753,0.287
"void mlx::core::rocm::gather_axis_kernel<float, int, 1, true, true>(float const*, int const*, float*, long, long, long, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, mlx::core::rocm::hip_array<long, 10>, int, int, long, long)",gather_scatter,,,2921,22.82,5114065,1751,0.287
"void mlx::core::rocm::binary_g<mlx::core::rocm::Multiply, hip_bfloat16, hip_bfloat16, long, 1>(hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, mlx::core::rocm::hip_array<long, 10>, int)",binary,,,2921,22.82,5085612,1741,0.285
"void mlx::core::rocm::row_reduce_simple_kernel<float, float, mlx::core::rocm::Sum, 4>(float const*, float*, unsigned long, int)",reduce,,,2921,22.82,5084396,1741,0.285
"void mlx::core::rocm::binary_vs<mlx::core::rocm::Maximum, float, float, unsigned int, 4>(float const*, float const*, float*, unsigned int)",binary,ssm_step,2067,2921,22.82,4927030,1687,0.276
"void mlx::core::rocm::Cf4IAsTypeADf4OSigmoidCEf4IBroadcastDBFf4IBroadcastBDGf4IAddEFHf4ONegativeG_VV_V2f4_6142509188972423790_strided<1, unsigned int, 8>(hip_bfloat16 const*, hip::std::array<long long, 1>, float const*, hip::std::array<long long, 1>, float*, float*, hip::std::array<int, 1>, unsigned int)",compiled,,,2921,22.82,4896259,1676,0.275
"void mlx::core::rocm::Ef4IBroadcastBCFf4IBroadcastCBGf4IAddEFHf4IBroadcastAGIf4IBroadcastGAJf4IDivideHIKf4IBroadcastJDLf4IBroadcastDJMf4OMultiplyKL_VVCS_f4f4f4_9971424201298772266_strided<1, unsigned int, 1>(float const*, hip::std::array<long long, 1>, float const*, hip::std::array<long long, 1>, float const*, float*, hip::std::array<int, 1>, unsigned int)",compiled,,,2921,22.82,4077494,1396,0.229
"void mlx::core::rocm::arg_reduce_partial<hip_bfloat16, mlx::core::rocm::ArgMax<hip_bfloat16>, 256>(hip_bfloat16 const*, hip_bfloat16*, unsigned int*, unsigned long, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, int, long, int, int)",arg_reduce,sampler_tail,2064,127,0.99,1228980,9677,0.069
"void mlx::core::rocm::copy_v<hip_bfloat16, hip_bfloat16, unsigned int, 4>(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",copy,sampler_tail,2064,127,0.99,429689,3383,0.024
"void mlx::core::rocm::gather_rows_kernel<hip_bfloat16, int>(hip_bfloat16 const*, int const*, hip_bfloat16*, long, unsigned int, int)",gather_scatter,,,254,1.98,423680,1668,0.024
"void mlx::core::rocm::gather_axis_kernel<hip_bfloat16, int, 1, true, true>(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, mlx::core::rocm::hip_array<long, 10>, int, int, long, long)",gather_scatter,sampler_tail,2064,127,0.99,365236,2876,0.020
"void mlx::core::rocm::scatter_axis_kernel<hip_bfloat16, int, false, 1, true, true>(hip_bfloat16 const*, int const*, hip_bfloat16*, long, long, long, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, mlx::core::rocm::hip_array<long, 10>, mlx::core::rocm::hip_array<long, 10>, int, int, long, long, long)",gather_scatter,sampler_tail,2064,127,0.99,304541,2398,0.017
"void mlx::core::rocm::arg_reduce_final<hip_bfloat16, mlx::core::rocm::ArgMax<hip_bfloat16>, 256>(hip_bfloat16 const*, unsigned int const*, unsigned int*, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, int, unsigned long, int)",arg_reduce,sampler_tail,2064,127,0.99,266088,2095,0.015
"void mlx::core::rocm::gather_rows_kernel<unsigned int, int>(unsigned int const*, int const*, unsigned int*, long, unsigned int, int)",gather_scatter,,,127,0.99,247700,1950,0.014
"void mlx::core::rocm::affine_dequantize_packed_kernel<hip_bfloat16, 4>(unsigned char const*, hip_bfloat16 const*, hip_bfloat16 const*, hip_bfloat16*, unsigned long, int)",dequantize,,,127,0.99,237600,1871,0.013
"void mlx::core::rocm::copy_v<unsigned int, int, unsigned int, 4>(unsigned int const*, int*, unsigned int)",copy,sampler_tail,2064,126,0.98,201576,1600,0.011
"void mlx::core::rocm::copy_s<hip_bfloat16, hip_bfloat16, unsigned int, 4>(hip_bfloat16 const*, hip_bfloat16*, unsigned int)",copy,,,12,0.09,16191,1349,0.001
"void mlx::core::rocm::copy_g_byval<hip_bfloat16, hip_bfloat16, long>(hip_bfloat16 const*, hip_bfloat16*, long, mlx::core::rocm::hip_array<int, 10>, mlx::core::rocm::hip_array<long, 10>, int)",copy,ssm_step,2067,2,0.02,3606,1803,0.000
Loading
Loading