diff --git a/README.md b/README.md index 8fa09065a..0b056026c 100644 --- a/README.md +++ b/README.md @@ -133,11 +133,11 @@ Command: `cactus benchmark` [optional `--ios` or `--android`] | Device | LLM | VLM | Transcribe | RAM | |--------|-----|-----|------------|---------------| | Mac M5 Max | 2964tps / 154tps | 0.09s / 168tps | 0.15s | 1348MB | -| Mac M4 Pro | 1947tps / 97tps | 0.25s / 108tps | 0.28s | 1225MB | +| Mac M4 Pro | 1963tps / 101tps | 0.25s / 112tps | 0.21s | 1225MB | | Mac M3 Pro | 1294tps / 64tps | 0.40s / 72tps | 0.37s | 735MB | | iPad/Vision Pro M5 | 1336tps / 71tps | 0.25s / 80tps | 0.27s | 703MB | | iPhone 17 Pro | 729tps / 37tps | 0.5s / 39tps | 0.51s | 644MB | -| iPhone 15 Pro | 511tps / 25tps | 1.16s / 27tps | 1.40s | 635MB | +| iPhone 15 Pro | 517tps / 26tps | 1.15s / 27tps | 0.82s | 633MB | N/B: With 1k-context prefill and decode for 100 runs on M5 Max - `LFM2.5-VL-1.6B` = 289toks/sec diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 01c3b8b4d..1c4a724c5 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -217,32 +217,20 @@ kernel void cq4_gemv( if (lane==0) y[n]=(half)acc; } -#define CQ4_NR 2u -kernel void cq4_gemv_mr( - device const half* code [[buffer(0)]], - device const uchar* packed [[buffer(1)]], - device const half* codebook [[buffer(2)]], - device const half* norms [[buffer(3)]], - device half* y [[buffer(4)]], - constant uint& gs [[buffer(5)]], - constant uint& num_groups [[buffer(6)]], - constant uint& pgb [[buffer(7)]], - constant uint& N [[buffer(8)]], - constant uint& il [[buffer(9)]], - uint tg [[threadgroup_position_in_grid]], - uint sgid [[simdgroup_index_in_threadgroup]], - uint lane [[thread_index_in_simdgroup]], - uint tl [[thread_index_in_threadgroup]]) +template +static inline void cq4_gemv_mr_impl( + device const half* code, device const uchar* packed, device const half* codebook, + device const half* norms, device half* y, uint gs, uint num_groups, uint pgb, uint N, uint il, + threadgroup float* cb, uint tg, uint sgid, uint lane, uint tl) { - threadgroup float cb[16]; if (tl<16) cb[tl]=(float)codebook[tl]; threadgroup_barrier(mem_flags::mem_threadgroup); - uint n0 = (tg*ROWS + sgid)*CQ4_NR; + uint n0 = (tg*ROWS + sgid)*NR; if (n0>=N) return; uint K = num_groups*gs; - float acc[CQ4_NR]; + float acc[NR]; #pragma clang loop unroll(full) - for (uint r=0;r>2)*num_groups+g)*4u*(size_t)pgb + (n0&3u)*4u + (off>>4)*32u; - uint2 Aw = ((device const uint2*)pan)[0]; - uint2 Bw = ((device const uint2*)(pan+16u))[0]; + uint Aw[NR], Bw[NR]; + if (NR == 4u) { + uint4 A = ((device const uint4*)pan)[0]; + uint4 B = ((device const uint4*)(pan+16u))[0]; + Aw[0]=A.x; Aw[1]=A.y; Aw[2]=A.z; Aw[3]=A.w; + Bw[0]=B.x; Bw[1]=B.y; Bw[2]=B.z; Bw[3]=B.w; + } else { + uint2 A = ((device const uint2*)pan)[0]; + uint2 B = ((device const uint2*)(pan+16u))[0]; + Aw[0]=A.x; Bw[0]=B.x; + if (NR>1u){ Aw[1]=A.y; Bw[1]=B.y; } + } + device const half* nb = norms + ((((size_t)(n0>>2)*num_groups+g)<<2) + (n0&3u)); #pragma clang loop unroll(full) - for (uint r=0;r>2)*num_groups+g)<<2) + (n0&3u) + r]*p; + for (uint r=0;r(code, packed, codebook, norms, y, gs, num_groups, pgb, N, il, cb, tg, sgid, lane, tl); } + +CQ4_MR_KERNEL(cq4_gemv_mr, 2u) +CQ4_MR_KERNEL(cq4_gemv_mr4, 4u) + struct TGU { uint K, ng, oswi; }; kernel void cq4_transform_gemv( device const half* x [[buffer(0)]], @@ -435,40 +448,28 @@ kernel void cq4_gemv_cat( } } -kernel void cq_gemv_mr_lowbit( - device const half* code [[buffer(0)]], - device const uchar* packed [[buffer(1)]], - device const half* codebook [[buffer(2)]], - device const half* norms [[buffer(3)]], - device half* y [[buffer(4)]], - constant uint& gs [[buffer(5)]], - constant uint& num_groups [[buffer(6)]], - constant uint& pgb [[buffer(7)]], - constant uint& N [[buffer(8)]], - constant uint& bits [[buffer(9)]], - constant uint& il [[buffer(10)]], - uint tg [[threadgroup_position_in_grid]], - uint sgid [[simdgroup_index_in_threadgroup]], - uint lane [[thread_index_in_simdgroup]], - uint tl [[thread_index_in_threadgroup]]) +template +static inline void cq_gemv_mr_lowbit_impl( + device const half* code, device const uchar* packed, device const half* codebook, + device const half* norms, device half* y, uint gs, uint num_groups, uint pgb, uint N, + uint bits, uint il, threadgroup float* cb, uint tg, uint sgid, uint lane, uint tl) { - threadgroup float cb[8]; uint cbn = 1u << bits; if (tl < cbn) cb[tl] = (float)codebook[tl]; threadgroup_barrier(mem_flags::mem_threadgroup); - uint n0 = (tg*ROWS + sgid)*CQ4_NR; + uint n0 = (tg*ROWS + sgid)*NR; if (n0 >= N) return; uint K = num_groups*gs; - float acc[CQ4_NR]; + float acc[NR]; #pragma clang loop unroll(full) - for (uint r=0;r> 3; #pragma clang loop unroll(full) - for (uint r=0;r(code, packed, codebook, norms, y, gs, num_groups, pgb, N, bits, il, cb, tg, sgid, lane, tl); } + +CQ_LOWBIT_KERNEL(cq_gemv_mr_lowbit, 2u) +CQ_LOWBIT_KERNEL(cq_gemv_mr_lowbit4, 4u) + kernel void cq_act_quant_i8( device const half* code [[buffer(0)]], device char* act_i8 [[buffer(1)]], @@ -2408,40 +2424,71 @@ kernel void attn_decode_fused_i8( } } - float qreg[16]; - { uint i=0; for (uint d=lane; d +static inline void attn_decode_gqa_impl( + device const half* q, device const half* knew, device const half* vnew, + device const char* kc, device const char* vc, device const float* ks, device const float* vs, + device half* out, uint num_kv_heads, uint head_dim, uint v_hdim, + uint history_len, float scale, uint kv_start, uint kv_end, + device float* part_o, device float* part_ml, uint nwg, uint hg, uint tg, uint lane) +{ + const uint per_kv = hg * nwg; + const uint kvh = tg / per_kv; + const uint rem = tg % per_kv; + const uint group = rem / nwg; + const uint w = rem % nwg; + const uint h0 = kvh * (hg * BG) + group * BG; + const uint ngK = head_dim / 32u; + const uint ngV = v_hdim / 32u; + const uint lane8 = lane / 8u; + + float4 qf[BG][UF]; + for (uint g = 0; g < BG; ++g) + for (uint u = 0; u < UF; ++u) + qf[g][u] = (float4)(*(device const half4*)(q + (size_t)(h0+g)*head_dim + 128u*u + 4u*lane)); + + float4 of[BG][UF]; + float m_i[BG], l_i[BG]; + for (uint g = 0; g < BG; ++g) { m_i[g] = -INFINITY; l_i[g] = 0.0f; + for (uint u = 0; u < UF; ++u) of[g][u] = float4(0.0f); } + + for (uint k = kv_start + w; k < kv_end; k += nwg) { + float4 kf[UF], vf[UF]; + if (k < history_len) { + size_t kb = ((size_t)k*num_kv_heads + kvh)*head_dim; + size_t vb = ((size_t)k*num_kv_heads + kvh)*v_hdim; + size_t ksb = ((size_t)k*num_kv_heads + kvh)*ngK; + size_t vsb = ((size_t)k*num_kv_heads + kvh)*ngV; + for (uint u = 0; u < UF; ++u) { + char4 ck = *(device const char4*)(kc + kb + 128u*u + 4u*lane); + char4 cv = *(device const char4*)(vc + vb + 128u*u + 4u*lane); + float ksc = ks[ksb + 4u*u + lane8]; + float vsc = vs[vsb + 4u*u + lane8]; + kf[u] = float4((float)ck.x,(float)ck.y,(float)ck.z,(float)ck.w) * ksc; + vf[u] = float4((float)cv.x,(float)cv.y,(float)cv.z,(float)cv.w) * vsc; + } + } else { + uint kn = k - history_len; + size_t kb = ((size_t)kn*num_kv_heads + kvh)*head_dim; + size_t vb = ((size_t)kn*num_kv_heads + kvh)*v_hdim; + for (uint u = 0; u < UF; ++u) { + kf[u] = (float4)(*(device const half4*)(knew + kb + 128u*u + 4u*lane)); + vf[u] = (float4)(*(device const half4*)(vnew + vb + 128u*u + 4u*lane)); + } + } + for (uint g = 0; g < BG; ++g) { + float d = 0.0f; + for (uint u = 0; u < UF; ++u) d += dot(qf[g][u], kf[u]); + float s = simd_sum(d) * scale; + float m_new = max(m_i[g], s); + float resc = exp(m_i[g] - m_new); + float p = exp(s - m_new); + l_i[g] = l_i[g]*resc + p; + for (uint u = 0; u < UF; ++u) of[g][u] = of[g][u]*resc + p*vf[u]; + m_i[g] = m_new; + } + } + + if (nwg == 1u) { + for (uint g = 0; g < BG; ++g) { + float inv = l_i[g] > 0.0f ? 1.0f/l_i[g] : 0.0f; + for (uint u = 0; u < UF; ++u) + *(device half4*)(out + (size_t)(h0+g)*v_hdim + 128u*u + 4u*lane) = (half4)(of[g][u]*inv); + } + } else { + for (uint g = 0; g < BG; ++g) { + uint slot = (h0 + g)*nwg + w; + if (lane == 0) { part_ml[(size_t)slot*2] = m_i[g]; part_ml[(size_t)slot*2 + 1] = l_i[g]; } + for (uint u = 0; u < UF; ++u) + *(device float4*)(part_o + (size_t)slot*v_hdim + 128u*u + 4u*lane) = of[g][u]; + } + } +} + +#define ATTN_DECODE_GQA(BG, UF) \ +kernel void attn_decode_i8_gqa_##BG##_##UF( \ + device const half* q [[buffer(0)]], device const half* knew [[buffer(1)]], \ + device const half* vnew [[buffer(2)]], device const char* kc [[buffer(3)]], \ + device const char* vc [[buffer(4)]], device const float* ks [[buffer(5)]], \ + device const float* vs [[buffer(6)]], device half* out [[buffer(7)]], \ + constant uint& num_q_heads [[buffer(8)]], constant uint& num_kv_heads [[buffer(9)]], \ + constant uint& head_dim [[buffer(10)]], constant uint& v_hdim [[buffer(11)]], \ + constant uint& history_len [[buffer(12)]], constant float& scale [[buffer(13)]], \ + constant uint& kv_start [[buffer(14)]], constant uint& kv_end [[buffer(15)]], \ + device float* part_o [[buffer(16)]], device float* part_ml [[buffer(17)]], \ + constant uint& nwg [[buffer(18)]], constant uint& hg [[buffer(19)]], \ + uint tg [[threadgroup_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) \ +{ attn_decode_gqa_impl(q, knew, vnew, kc, vc, ks, vs, out, num_kv_heads, head_dim, v_hdim, \ + history_len, scale, kv_start, kv_end, part_o, part_ml, nwg, hg, tg, lane); } + +ATTN_DECODE_GQA(1,1) ATTN_DECODE_GQA(2,1) ATTN_DECODE_GQA(3,1) ATTN_DECODE_GQA(4,1) +ATTN_DECODE_GQA(5,1) ATTN_DECODE_GQA(6,1) ATTN_DECODE_GQA(7,1) ATTN_DECODE_GQA(8,1) +ATTN_DECODE_GQA(1,2) ATTN_DECODE_GQA(2,2) ATTN_DECODE_GQA(3,2) ATTN_DECODE_GQA(4,2) + kernel void attn_prefill_i8( device const half* q [[buffer(0)]], device const half* knew [[buffer(1)]], diff --git a/cactus-kernels/src/matmul.cpp b/cactus-kernels/src/matmul.cpp index 2873ac911..851795915 100644 --- a/cactus-kernels/src/matmul.cpp +++ b/cactus-kernels/src/matmul.cpp @@ -2275,6 +2275,10 @@ static void cactus_quant_interleaved4_gemv_blocks( const uint8_t* p1 = packed_interleaved + ((nb + 1) * num_groups + g) * panel_bytes; const int8_t* a_grp = act_i8 + static_cast(g) * gs; const float a_sc = act_scales[g]; + if (g + 3 < num_groups) { + __builtin_prefetch(p0 + 3 * panel_bytes, 0, 0); + __builtin_prefetch(p1 + 3 * panel_bytes, 0, 0); + } int32x4_t d0a = vdupq_n_s32(0), d0b = vdupq_n_s32(0); int32x4_t d1a = vdupq_n_s32(0), d1b = vdupq_n_s32(0); diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 0c4a5f236..829a11fb6 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -37,7 +37,7 @@ id psoTsimd = nil; id psoTbatch = nil; id psoMega = nil, psoSwiT = nil, psoCat = nil; - id psoG = nil, psoGmr = nil, psoGmrLow = nil; + id psoG = nil, psoGmr = nil, psoGmr4 = nil, psoGmrLow = nil, psoGmrLow4 = nil; id psoActQ = nil, psoG2i8 = nil; id psoTm = nil; id psoGmma = nil, psoGdense = nil; @@ -47,6 +47,7 @@ id psoCopy=nil, psoBinary=nil, psoScalar=nil, psoUnary=nil, psoRms=nil, psoSwiglu=nil, psoRmsAdd=nil, psoRmsAddScale=nil; id psoCF16F32=nil, psoCF32F16=nil, psoCI8F16=nil, psoCF16I8=nil; id psoAttn=nil, psoAttnC=nil, psoAttnF=nil, psoStrided=nil, psoScatter=nil, psoBcast=nil, psoKvAppend=nil; + id psoAttnGqa[9][3] = {}; id psoAttnPre=nil, psoAttnPreMma2=nil, psoAttnPreHd256=nil; id psoKvAppendM=nil, psoKvAppendRingM=nil; id psoSlideS=nil, psoSlideR=nil, psoSlideRM=nil; @@ -86,11 +87,11 @@ if (!p) fprintf(stderr,"[cactus-metal] pipeline '%s' failed: %s\n", name, e?[[e localizedDescription] UTF8String]:"function not found"); return p; }; - psoT=pso("cq4_transform"); psoTsimd=pso("cq4_transform_simd"); psoG=pso("cq4_gemv"); psoTbatch=pso("cq4_transform_batch"); psoGmr=pso("cq4_gemv_mr"); + psoT=pso("cq4_transform"); psoTsimd=pso("cq4_transform_simd"); psoG=pso("cq4_gemv"); psoTbatch=pso("cq4_transform_batch"); psoGmr=pso("cq4_gemv_mr"); psoGmr4=pso("cq4_gemv_mr4"); psoMega=pso("cq4_transform_gemv"); psoSwiT=pso("cq4_swiglu_transform"); psoCat=pso("cq4_gemv_cat"); psoTm=pso("cq4_transform_m"); psoGmma=pso("cq4_gemm_mma"); psoGdense=pso("cq4_gemm_dense_f16"); - psoGmrLow=pso("cq_gemv_mr_lowbit"); + psoGmrLow=pso("cq_gemv_mr_lowbit"); psoGmrLow4=pso("cq_gemv_mr_lowbit4"); psoActQ=pso("cq_act_quant_i8"); psoG2i8=pso("cq2_gemv_i8"); psoRotW=pso("lmhead_rotate_wide"); psoEmbOW=pso("emb_ortho_wide"); psoRmsAddRms=pso("rms_norm_add_rms_f16"); psoEmbH=pso("emb_hadamard"); @@ -101,6 +102,8 @@ psoCF16F32=pso("cast_f16_f32"); psoCF32F16=pso("cast_f32_f16"); psoCI8F16=pso("cast_i8_f16"); psoCF16I8=pso("cast_f16_i8"); psoAttn=pso("attn_decode_i8"); psoAttnC=pso("attn_decode_combine"); psoAttnF=pso("attn_decode_fused_i8"); + for (uint32_t b=1; b<=8; ++b) for (uint32_t u=1; u<=2; ++u) + if (!(u==2 && b>4)) psoAttnGqa[b][u]=pso(("attn_decode_i8_gqa_"+std::to_string(b)+"_"+std::to_string(u)).c_str()); psoStrided=pso("strided_copy_f16"); psoBcast=pso("bcast_binary_f16"); psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoAttnPreHd256=pso("attn_prefill_mma_hd256"); psoKvAppendM=pso("kv_append_i8_m"); psoKvAppendRingM=pso("kv_append_ring_i8_m"); @@ -774,8 +777,10 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu [g_enc dispatchThreadgroups:MTLSizeMake((N+7u)/8u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } + bool umr4 = !lowbit && ctx().psoGmr4 && rw.il != 0u && (N % (8u*4u) == 0u); + bool lowbit4 = lowbit && ctx().psoGmrLow4 && (N % (8u*4u) == 0u); bool umr = ctx().psoGmr && (N % g_mr_rows == 0u); - [g_enc setComputePipelineState:(lowbit?ctx().psoGmrLow:(umr?ctx().psoGmr:ctx().psoG))]; + [g_enc setComputePipelineState:(lowbit4?ctx().psoGmrLow4:(lowbit?ctx().psoGmrLow:(umr4?ctx().psoGmr4:(umr?ctx().psoGmr:ctx().psoG))))]; [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:1]; [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:2]; [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:3]; setBufAt(out, (size_t)N*2, 4); @@ -784,7 +789,8 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu if (lowbit) { [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc setBytes:&rw.il length:4 atIndex:10]; } else [g_enc setBytes:&rw.il length:4 atIndex:9]; uint32_t ROWS=8; - uint32_t grid = (lowbit || umr) ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; + uint32_t grid = (umr4 || lowbit4) ? (N+8u*4u-1u)/(8u*4u) + : ((lowbit || umr) ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS); [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; return true; } @@ -1525,15 +1531,37 @@ bool cactus_metal_encode_attention_i8( setCacheAt(ks, ks_bytes, 5); setCacheAt(vs, vs_bytes, 6); }; - const uint32_t T = 256, nsg = T / 32u; const uint32_t R = kv_end - kv_start; - uint32_t nwg = R / 24u; if (nwg < 1u) nwg = 1u; if (nwg > 32u) nwg = 32u; + const uint32_t gqa = num_q_heads / num_kv_heads; + const uint32_t uf = head_dim / 128u; + uint32_t bg = gqa, hg = 1u; + if (gqa % 4u == 0u) { bg = 4u; hg = gqa / 4u; } + else if (gqa % 3u == 0u) { bg = 3u; hg = gqa / 3u; } + else if (gqa % 2u == 0u) { bg = 2u; hg = gqa / 2u; } + if (uf >= 2u && bg == 4u) { bg = 2u; hg = gqa / 2u; } + const bool use_gqa = gqa >= 1u && uf >= 1u && uf <= 2u + && head_dim == v_hdim && head_dim == uf*128u + && bg <= 8u && ctx().psoAttnGqa[bg][uf]; + + uint32_t T, nsg, nwg; + if (use_gqa) { + T = 32u; nsg = 1u; + uint32_t units = num_kv_heads * hg; + uint32_t max_by_keys = R / 32u; if (max_by_keys < 1u) max_by_keys = 1u; + nwg = (256u + units - 1u) / units; + if (nwg > max_by_keys) nwg = max_by_keys; + if (nwg > 256u) nwg = 256u; if (nwg < 1u) nwg = 1u; + } else { + T = 256u; nsg = T / 32u; + nwg = R / 24u; if (nwg < 1u) nwg = 1u; if (nwg > 32u) nwg = 32u; + } + id partO = ctx().dummy, partML = ctx().dummy; if (nwg > 1u) { partO = recycled((size_t)num_q_heads*nwg*v_hdim*sizeof(float)); partML = recycled((size_t)num_q_heads*nwg*2*sizeof(float)); } - [g_enc setComputePipelineState:ctx().psoAttn]; + [g_enc setComputePipelineState:(use_gqa ? ctx().psoAttnGqa[bg][uf] : ctx().psoAttn)]; setInputs(); setBufAt(out, (size_t)num_q_heads*v_hdim*2, 7); [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; @@ -1542,8 +1570,10 @@ bool cactus_metal_encode_attention_i8( [g_enc setBytes:&kv_start length:4 atIndex:14]; [g_enc setBytes:&kv_end length:4 atIndex:15]; [g_enc setBuffer:partO offset:0 atIndex:16]; [g_enc setBuffer:partML offset:0 atIndex:17]; [g_enc setBytes:&nwg length:4 atIndex:18]; - [g_enc setThreadgroupMemoryLength:((size_t)nsg*v_hdim + 2*nsg)*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + if (use_gqa) [g_enc setBytes:&hg length:4 atIndex:19]; + else [g_enc setThreadgroupMemoryLength:((size_t)nsg*v_hdim + 2*nsg)*sizeof(float) atIndex:0]; + uint32_t base_tg = use_gqa ? (num_kv_heads*hg) : num_q_heads; + [g_enc dispatchThreadgroups:MTLSizeMake(base_tg*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; if (nwg > 1u) { [g_enc setComputePipelineState:ctx().psoAttnC]; [g_enc setBuffer:partO offset:0 atIndex:0]; [g_enc setBuffer:partML offset:0 atIndex:1];