diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index fc19b2ccd..9e605e3b5 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -949,7 +949,11 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n const auto& w1_0 = get_input(node, 3, nodes, map); if (!PrecisionTraits::is_cq(w1_0.precision) || w1_0.group_size == 0) return false; CactusQuantMatrix key = w1_0.to_cq_matrix(); - if (!cactus_metal_moe_cq4_ready(&key)) { + const bool cq2 = key.bits == 2u; + const bool cq4 = key.bits == 4u; + if (!cq2 && !cq4) return false; + const bool ready = cq2 ? cactus_metal_moe_cq2_ready(&key) : cactus_metal_moe_cq4_ready(&key); + if (!ready) { std::vector w1s(num_experts), w3s(num_experts), w2s(num_experts); for (size_t e = 0; e < num_experts; ++e) { const auto& b1 = get_input(node, 3 + e, nodes, map); @@ -962,12 +966,20 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n w3s[e] = b3.to_cq_matrix(); w2s[e] = b2.to_cq_matrix(); } - if (!cactus_metal_moe_cq4_build(w1s.data(), w3s.data(), w2s.data(), (uint32_t)num_experts)) + const bool built = cq2 + ? cactus_metal_moe_cq2_build(w1s.data(), w3s.data(), w2s.data(), (uint32_t)num_experts) + : cactus_metal_moe_cq4_build(w1s.data(), w3s.data(), w2s.data(), (uint32_t)num_experts); + if (!built) return false; } + if (cq2) { + return cactus_metal_encode_moe_gated_cq2(out.get_data(), hidden.get_data(), probs.get_data(), + topk.get_data(), &key, (uint32_t)num_experts, (uint32_t)top_k, (uint32_t)hidden.shape[0], + act, node.params.normalize_routing ? 1u : 0u, node.params.epsilon, node.params.scalar); + } return cactus_metal_encode_moe_gated_cq4(out.get_data(), hidden.get_data(), probs.get_data(), - topk.get_data(), &key, (uint32_t)num_experts, (uint32_t)top_k, (uint32_t)hidden.shape[0], - act, node.params.normalize_routing ? 1u : 0u, node.params.epsilon, node.params.scalar); + topk.get_data(), &key, (uint32_t)num_experts, (uint32_t)top_k, (uint32_t)hidden.shape[0], + act, node.params.normalize_routing ? 1u : 0u, node.params.epsilon, node.params.scalar); } case OpType::LAYERNORM: { if (node.input_ids.size() < 2 || out.shape.empty()) return false; diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 9505aad48..1c5eae6cd 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -94,6 +94,13 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void const void* topk, const CactusQuantMatrix* w1_0, uint32_t num_experts, uint32_t top_k, uint32_t tokens, uint32_t act, uint32_t normalize, float eps, float scaling); +bool cactus_metal_moe_cq2_ready(const CactusQuantMatrix* w1_0); +bool cactus_metal_moe_cq2_build(const CactusQuantMatrix* w1s, const CactusQuantMatrix* w3s, + const CactusQuantMatrix* w2s, uint32_t num_experts); +bool cactus_metal_encode_moe_gated_cq2(void* out, const void* hidden, const void* probs, + const void* topk, const CactusQuantMatrix* w1_0, + uint32_t num_experts, uint32_t top_k, uint32_t tokens, + uint32_t act, uint32_t normalize, float eps, float scaling); bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* code, const CactusQuantMatrix* W); bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W, float scale); diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 01c3b8b4d..9e019c0d4 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -85,6 +85,11 @@ static inline float cq4_dot64_ilw(half4 c0, half4 c1, half4 c2, half4 c3, + (float)c3.x*cb[(B>>4)&0xFu] + (float)c3.y*cb[(B>>12)&0xFu] + (float)c3.z*cb[(B>>20)&0xFu] + (float)c3.w*cb[B>>28]; } +static inline float cq2_dot4(half4 c, uchar w, threadgroup const float* cb) { + return (float)c.x*cb[w&3u] + (float)c.y*cb[(w>>2)&3u] + + (float)c.z*cb[(w>>4)&3u] + (float)c.w*cb[(w>>6)&3u]; +} + kernel void cq4_transform( device const half* x [[buffer(0)]], device const half* recip [[buffer(1)]], @@ -136,6 +141,38 @@ kernel void cq4_transform_simd( code[b+2]=(half)zmem[perm[k+2]]; code[b+3]=(half)zmem[perm[k+3]]; } +kernel void cq2_transform_f16( + device const half* x [[buffer(0)]], + device const half* recip [[buffer(1)]], + device const char* lsign [[buffer(2)]], + device const char* rsign [[buffer(3)]], + device const uint* perm [[buffer(4)]], + device half* code [[buffer(5)]], + constant uint& K [[buffer(6)]], + uint2 tgp [[threadgroup_position_in_grid]], + uint2 tp [[thread_position_in_threadgroup]], + threadgroup half* z [[threadgroup(0)]]) +{ + const uint t = tp.x; + const uint g = tgp.x, m = tgp.y; + const size_t base = (size_t)m*K + g*128u; + half v = (half)((half)(x[base+t] * recip[g*128u+t]) * (half)lsign[t]); + z[t] = v; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint h = 1u; h < 128u; h <<= 1u) { + if ((t & h) == 0u) { + half a = z[t], b = z[t+h]; + z[t] = (half)(a+b); + z[t+h] = (half)(a-b); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + const half inv = (half)0.08838834764831845f; + z[t] = (half)((half)(z[t] * inv) * (half)rsign[t]); + threadgroup_barrier(mem_flags::mem_threadgroup); + code[base+t] = z[perm[t]]; +} + kernel void cq4_transform_batch( device const half* x [[buffer(0)]], device const half* recip [[buffer(1)]], @@ -577,6 +614,43 @@ kernel void cq2_gemv_i8( if (lane == 0u) y[n] = (half)acc; } +kernel void cq2_gemm_i8( + device const char* act_i8 [[buffer(0)]], + device const float* act_scales [[buffer(1)]], + device const uchar* packed [[buffer(2)]], + device const char* cb_i8 [[buffer(3)]], + device const half* norms [[buffer(4)]], + device half* y [[buffer(5)]], + constant uint& num_groups [[buffer(6)]], + constant uint& N [[buffer(7)]], + constant float& cb_scale [[buffer(8)]], + uint2 tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]]) +{ + threadgroup int cb[4]; + if (tl < 4u) cb[tl] = (int)cb_i8[tl]; + threadgroup_barrier(mem_flags::mem_threadgroup); + const uint m = tg.y; + const uint n = tg.x*8u + sgid; + if (n >= N) return; + float acc = 0.f; + for (uint g = 0u; g < num_groups; ++g) { + uchar b = packed[((size_t)(n>>2)*num_groups + g)*128u + + (lane>>2)*16u + (lane&3u)*4u + (n&3u)]; + device const char* a = act_i8 + ((size_t)m*num_groups + g)*128u + lane*4u; + int partial = (int)a[0]*cb[b & 3u] + (int)a[1]*cb[(b >> 2) & 3u] + + (int)a[2]*cb[(b >> 4) & 3u] + (int)a[3]*cb[(b >> 6) & 3u]; + int idot = simd_sum(partial); + if (lane == 0u) { + float ns = (float)norms[(((size_t)(n>>2)*num_groups+g)<<2) + (n&3u)] * cb_scale; + acc = fma((float)idot, ns * act_scales[(size_t)m*num_groups+g], acc); + } + } + if (lane == 0u) y[(size_t)m*N+n] = (half)acc; +} + kernel void cq4_transform_m( device const half* x [[buffer(0)]], device const half* recip [[buffer(1)]], @@ -1039,6 +1113,7 @@ kernel void rms_norm_f16(device const half* in [[buffer(0)]], device const half* float inv = 1.0f/sqrt(red[0]/(float)dim + eps); for (uint i=t;i= topk_k; + const uint slot = use_w3 ? tgp.y-topk_k : tgp.y; + const uint e = (uint)(topk[(size_t)tok*topk_k + slot] + 0.5f); + const uint es = use_w3 ? estride3 : estride1; + device const half* recip = (device const half*)((device const uchar*)(use_w3 ? recip_c3 : recip_c1) + (size_t)e*es); + device const char* ls = (use_w3 ? lsign_c3 : lsign_c1) + (size_t)e*es; + device const char* rs = (use_w3 ? rsign_c3 : rsign_c1) + (size_t)e*es; + device const uint* pm = (device const uint*)((device const uchar*)(use_w3 ? perm_c3 : perm_c1) + (size_t)e*es); + const uint k = g*128u + t; + z[t] = (half)((half)(x[(size_t)tok*K + k]*recip[k])*(half)ls[t]); + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint h = 1u; h < 128u; h <<= 1u) { + if ((t & h) == 0u) { + half a = z[t], b = z[t+h]; + z[t] = (half)(a+b); + z[t+h] = (half)(a-b); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + const half inv = (half)0.08838834764831845f; + z[t] = (half)((half)(z[t]*inv)*(half)rs[t]); + threadgroup_barrier(mem_flags::mem_threadgroup); + device half* co = (use_w3 ? code3 : code1) + ((size_t)tok*topk_k + slot)*K + g*128u; + co[t] = z[pm[t]]; +} + +kernel void cq2_moe_gemv_up( + device const half* code1 [[buffer(0)]], + device const half* code3 [[buffer(1)]], + device const float* topk [[buffer(2)]], + device const uchar* pk1 [[buffer(3)]], + device const half* nm1 [[buffer(4)]], + device const half* cb1 [[buffer(5)]], + device const uchar* pk3 [[buffer(6)]], + device const half* nm3 [[buffer(7)]], + device const half* cb3 [[buffer(8)]], + device half* y [[buffer(9)]], + constant uint& K [[buffer(10)]], + constant uint& N [[buffer(11)]], + constant uint& act [[buffer(12)]], + constant uint& topk_k [[buffer(13)]], + constant uint& estride1 [[buffer(14)]], + constant uint& estride3 [[buffer(15)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]]) +{ + const uint slot = tgp.y, tok = tgp.z, slotg = tok*topk_k + slot; + const uint e = (uint)(topk[(size_t)tok*topk_k + slot] + 0.5f), ng = K/128u; + threadgroup float c1[4], c3[4]; + if (tl < 4u) { + c1[tl] = (float)((device const half*)((device const uchar*)cb1 + (size_t)e*estride1))[tl]; + c3[tl] = (float)((device const half*)((device const uchar*)cb3 + (size_t)e*estride3))[tl]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + const uint n0 = (tgp.x*8u + sgid)*4u; + if (n0 >= N) return; + float gv[4] = {0,0,0,0}, uv[4] = {0,0,0,0}; + for (uint which = 0u; which < 2u; ++which) { + device const uchar* pb = which == 0u ? pk1 + (size_t)e*estride1 : pk3 + (size_t)e*estride3; + device const half* nb = which == 0u + ? (device const half*)((device const uchar*)nm1 + (size_t)e*estride1) + : (device const half*)((device const uchar*)nm3 + (size_t)e*estride3); + device const half* cd = which == 0u ? code1 + (size_t)slotg*K : code3 + (size_t)slotg*K; + threadgroup float* cb = which == 0u ? c1 : c3; + float acc[4] = {0,0,0,0}; + for (uint g = 0u; g < ng; ++g) { + half4 cv = *(device const half4*)(cd + (size_t)g*128u + lane*4u); + #pragma clang loop unroll(full) + for (uint r = 0u; r < 4u; ++r) { + uint n = n0+r; + uchar w = pb[((size_t)(n>>2)*ng+g)*128u + (lane>>2)*16u + (lane&3u)*4u + (n&3u)]; + float norm = (float)nb[(((size_t)(n>>2)*ng+g)<<2) + (n&3u)]; + acc[r] += norm*cq2_dot4(cv, w, cb); + } + } + #pragma clang loop unroll(full) + for (uint r = 0u; r < 4u; ++r) { + float a = simd_sum(acc[r]); + if (which == 0u) gv[r] = a; + else uv[r] = a; + } + } + if (lane == 0u) { + device half* ys = y + (size_t)slotg*N; + for (uint r = 0u; r < 4u; ++r) { + float gh = (float)(half)gv[r]; + float av = act == 1u ? gelu_tanh(gh) : gh/(1.0f + precise::exp(-gh)); + ys[n0+r] = (half)((float)(half)av*(float)(half)uv[r]); + } + } +} + +kernel void cq2_moe_gemv_down_acc( + device const half* code2 [[buffer(0)]], + device const float* topk [[buffer(1)]], + device const half* probs [[buffer(2)]], + device const uchar* pk2 [[buffer(3)]], + device const half* nm2 [[buffer(4)]], + device const half* cb2 [[buffer(5)]], + device half* out [[buffer(6)]], + constant uint& K [[buffer(7)]], + constant uint& N [[buffer(8)]], + constant uint& topk_k [[buffer(9)]], + constant uint& normalize [[buffer(10)]], + constant float& eps [[buffer(11)]], + constant float& scaling [[buffer(12)]], + constant uint& E [[buffer(13)]], + constant uint& estride [[buffer(14)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]], + threadgroup float* cbs [[threadgroup(0)]]) +{ + const uint tok = tgp.z; + device const float* tk = topk + (size_t)tok*topk_k; + device const half* pr = probs + (size_t)tok*E; + uint order[16]; + for (uint j = 0u; j < topk_k; ++j) order[j] = j; + for (uint a = 1u; a < topk_k; ++a) { + uint v = order[a]; + uint ev = (uint)(tk[v] + 0.5f); + uint b = a; + while (b > 0u && (uint)(tk[order[b-1u]] + 0.5f) > ev) { + order[b] = order[b-1u]; + --b; + } + order[b] = v; + } + if (tl < topk_k*4u) { + uint s = tl >> 2, ci = tl & 3u; + uint e = (uint)(tk[s] + 0.5f); + cbs[tl] = (float)((device const half*)((device const uchar*)cb2 + (size_t)e*estride))[ci]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + float denom = 1.0f; + if (normalize != 0u) { + float s = 0.0f; + for (uint j = 0u; j < topk_k; ++j) s += (float)pr[(uint)(tk[j] + 0.5f)]; + denom = s + eps; + } + const uint ng = K/128u; + const uint n0 = (tgp.x*8u + sgid)*2u; + if (n0 >= N) return; + half res[2] = {(half)0, (half)0}; + for (uint j = 0u; j < topk_k; ++j) { + uint slot = order[j]; + uint e = (uint)(tk[slot] + 0.5f); + float p = (float)pr[e]; + if (p <= 0.0f) continue; + float rw = p/denom*scaling; + device const uchar* pb = pk2 + (size_t)e*estride; + device const half* nb = (device const half*)((device const uchar*)nm2 + (size_t)e*estride); + device const half* cd = code2 + ((size_t)tok*topk_k + slot)*K; + threadgroup const float* cb = cbs + (size_t)slot*4u; + float acc[2] = {0,0}; + for (uint g = 0u; g < ng; ++g) { + half4 cv = *(device const half4*)(cd + (size_t)g*128u + lane*4u); + #pragma clang loop unroll(full) + for (uint r = 0u; r < 2u; ++r) { + uint n = n0+r; + uchar w = pb[((size_t)(n>>2)*ng+g)*128u + (lane>>2)*16u + (lane&3u)*4u + (n&3u)]; + float norm = (float)nb[(((size_t)(n>>2)*ng+g)<<2) + (n&3u)]; + acc[r] += norm*cq2_dot4(cv, w, cb); + } + } + for (uint r = 0u; r < 2u; ++r) { + float a = simd_sum(acc[r]); + res[r] = (half)((float)res[r] + (float)(half)a*rw); + } + } + if (lane == 0u) { + device half* ys = out + (size_t)tok*N; + for (uint r = 0u; r < 2u; ++r) ys[n0+r] = res[r]; + } +} + kernel void cq4_moe_transform( device const half* x [[buffer(0)]], device const float* topk [[buffer(1)]], diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 0c4a5f236..0ee4de3c3 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -38,7 +38,7 @@ id psoTbatch = nil; id psoMega = nil, psoSwiT = nil, psoCat = nil; id psoG = nil, psoGmr = nil, psoGmrLow = nil; - id psoActQ = nil, psoG2i8 = nil; + id psoActQ = nil, psoG2i8 = nil, psoG2mi8 = nil, psoT2f16 = nil; id psoTm = nil; id psoGmma = nil, psoGdense = nil; id psoRotW = nil, psoEmbOW = nil, psoRmsAddRms = nil; @@ -66,6 +66,7 @@ id psoConvCacheAppend=nil, psoRelPosBias=nil, psoGemvBias=nil; id psoTopkRows=nil, psoMoeT=nil, psoMoeUp=nil; id psoMoeT2=nil, psoMoeDownAcc=nil, psoRopePair=nil; + id psoMoeT2Low=nil, psoMoeTLow=nil, psoMoeUpLow=nil, psoMoeDownLow=nil; id psoRmsScale=nil, psoSoftmaxTopk=nil, psoRopePairRms=nil; id psoRms2AddClip=nil, psoDeltanet=nil, psoDeltanetPre=nil; id dummy=nil; @@ -92,6 +93,7 @@ psoGdense=pso("cq4_gemm_dense_f16"); psoGmrLow=pso("cq_gemv_mr_lowbit"); psoActQ=pso("cq_act_quant_i8"); psoG2i8=pso("cq2_gemv_i8"); + psoG2mi8=pso("cq2_gemm_i8"); psoT2f16=pso("cq2_transform_f16"); psoRotW=pso("lmhead_rotate_wide"); psoEmbOW=pso("emb_ortho_wide"); psoRmsAddRms=pso("rms_norm_add_rms_f16"); psoEmbH=pso("emb_hadamard"); psoEmbOm=pso("emb_ortho_m"); psoEmbHm=pso("emb_hadamard_m"); @@ -131,6 +133,8 @@ psoTopkRows=pso("topk_rows_f16"); psoMoeT=pso("cq4_moe_transform"); psoMoeUp=pso("cq4_moe_gemv_up"); psoMoeT2=pso("cq4_moe_transform2"); psoMoeDownAcc=pso("cq4_moe_gemv_down_acc"); psoRopePair=pso("rope_pair_f16"); + psoMoeTLow=pso("cq2_moe_transform_f16"); psoMoeT2Low=pso("cq2_moe_transform2_f16"); + psoMoeUpLow=pso("cq2_moe_gemv_up"); psoMoeDownLow=pso("cq2_moe_gemv_down_acc"); psoRmsScale=pso("rms_norm_scale_f16"); psoSoftmaxTopk=pso("softmax_topk_f16"); psoRopePairRms=pso("rope_pair_rms_f16"); psoRms2AddClip=pso("rms2_add_clip_f16"); psoDeltanet=pso("gated_deltanet_decode_f16"); psoDeltanetPre=pso("gated_deltanet_prefill_f16"); @@ -192,14 +196,15 @@ static uint64_t resident_key(const CactusQuantMatrix* W) { std::unordered_map g_resident_emb; std::mutex g_resident_mu; -struct MoESet4 { +struct MoESet { id pk=nil, nm=nil, rc=nil, ls=nil, rs=nil, pm=nil, cb=nil; size_t pk_off=0, nm_off=0, rc_off=0, ls_off=0, rs_off=0, pm_off=0, cb_off=0; uint32_t K=0, N=0; uint32_t stride=0; + uint32_t bits=0; }; -struct MoECat4 { MoESet4 w1, w3, w2; bool ok=false; }; -std::unordered_map g_moe_cat; +struct MoECat { MoESet w1, w3, w2; bool ok=false; }; +std::unordered_map g_moe_cat; std::pair, size_t> wrapHostPtr(const void* p, size_t bytes); @@ -749,13 +754,20 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu g_code_buf = owned_shared(code_bytes); if (!g_code_buf) return false; ensureEncoder(); - bool simdT = (gs==128u && ctx().psoTsimd); - [g_enc setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; setBufAt(lhs, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:1]; [g_enc setBuffer:rw.lsign offset:rw.ls_off atIndex:2]; [g_enc setBuffer:rw.rsign offset:rw.rs_off atIndex:3]; [g_enc setBuffer:rw.perm offset:rw.pm_off atIndex:4]; [g_enc setBuffer:g_code_buf offset:0 atIndex:5]; - [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; + if (bits == 2u && gs == 128u && ctx().psoT2f16) { + [g_enc setComputePipelineState:ctx().psoT2f16]; + [g_enc setBytes:&K length:4 atIndex:6]; + [g_enc setThreadgroupMemoryLength:128u*sizeof(__fp16) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + } else { + bool simdT = (gs==128u && ctx().psoTsimd); + [g_enc setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; + [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; + } if (lowbit && bits == 2u && gs == 128u && ctx().psoActQ && ctx().psoG2i8 && rw.cb_i8) { id actq = recycled((size_t)ng*gs); id ascl = recycled((size_t)ng*sizeof(float)); @@ -924,13 +936,13 @@ bool moe_component_offsets(const CactusQuantMatrix& W, uintptr_t& lo, uintptr_t& const uintptr_t pg = 16384; struct Span { const void* p; size_t len; }; Span spans[7] = { - { W.packed_indices, (size_t)W.N*W.num_groups*64u }, + { W.packed_indices, (size_t)W.N*W.num_groups*((W.group_size*W.bits+7u)/8u) }, { W.norms, (size_t)W.N*W.num_groups*2 }, { W.input_scale_recip, (size_t)W.K*2 }, { W.left_signs, 128 }, { W.right_signs, 128 }, { W.permutation, 512 }, - { W.codebook, 32 }, + { W.codebook, (size_t)(1u< ab = [ctx().dev newBufferWithBytesNoCopy:arena length:total - options:MTLResourceStorageModeShared deallocator:nil]; + options:MTLResourceStorageModeShared deallocator:deallocator]; if (!ab) { munmap(arena, total); return false; } CactusThreading::parallel_for(count, CactusThreading::ParallelConfig(1, 1), [&](size_t start, size_t end) { @@ -989,7 +1003,7 @@ bool moe_build_set4_zerocopy(MoESet4& S, const CactusQuantMatrix* Ws, uint32_t c acc += *((const volatile uint8_t*)arena + e*slot + o); (void)acc; }); - S.K=Ws[0].K; S.N=Ws[0].N; + S.K=Ws[0].K; S.N=Ws[0].N; S.bits=Ws[0].bits; S.pk=S.nm=S.rc=S.ls=S.rs=S.pm=S.cb=ab; S.pk_off=offs[0]; S.nm_off=offs[1]; S.rc_off=offs[2]; S.ls_off=offs[3]; S.rs_off=offs[4]; S.pm_off=offs[5]; S.cb_off=offs[6]; @@ -997,21 +1011,37 @@ bool moe_build_set4_zerocopy(MoESet4& S, const CactusQuantMatrix* Ws, uint32_t c return true; } -bool moe_build_set4(MoESet4& S, const CactusQuantMatrix* Ws, uint32_t count) { +bool moe_build_set4(MoESet& S, const CactusQuantMatrix* Ws, uint32_t count) { const uint32_t K=Ws[0].K, N=Ws[0].N, gs=Ws[0].group_size, ng=Ws[0].num_groups; if (gs != 128u || Ws[0].bits != 4u || (N % 4u) != 0u || (K % 128u) != 0u) return false; for (uint32_t e = 0; e < count; ++e) { const CactusQuantMatrix* W = &Ws[e]; if (!quant_fast_eligible(W) || W->K != K || W->N != N || W->num_groups != ng) return false; } - return moe_build_set4_zerocopy(S, Ws, count); + return moe_build_set_zerocopy(S, Ws, count); +} + +bool moe_build_set2(MoESet& S, const CactusQuantMatrix* Ws, uint32_t count) { + const uint32_t K=Ws[0].K, N=Ws[0].N, gs=Ws[0].group_size, ng=Ws[0].num_groups; + if (gs != 128u || Ws[0].bits != 2u || (N % 4u) != 0u || (K % 128u) != 0u) return false; + for (uint32_t e = 0; e < count; ++e) { + const CactusQuantMatrix* W = &Ws[e]; + if (!quant_lowbit_eligible(W) || W->bits != 2u || W->K != K || W->N != N || W->num_groups != ng) return false; + } + return moe_build_set_zerocopy(S, Ws, count); } } bool cactus_metal_moe_cq4_ready(const CactusQuantMatrix* w1_0) { std::lock_guard lk(g_resident_mu); auto it = g_moe_cat.find(resident_key(w1_0)); - return it != g_moe_cat.end() && it->second.ok; + return it != g_moe_cat.end() && it->second.ok && it->second.w1.bits == 4u; +} + +bool cactus_metal_moe_cq2_ready(const CactusQuantMatrix* w1_0) { + std::lock_guard lk(g_resident_mu); + auto it = g_moe_cat.find(resident_key(w1_0)); + return it != g_moe_cat.end() && it->second.ok && it->second.w1.bits == 2u; } bool cactus_metal_moe_cq4_build(const CactusQuantMatrix* w1s, const CactusQuantMatrix* w3s, @@ -1022,30 +1052,58 @@ bool cactus_metal_moe_cq4_build(const CactusQuantMatrix* w1s, const CactusQuantM { std::lock_guard lk(g_resident_mu); auto it = g_moe_cat.find(key); - if (it != g_moe_cat.end()) return it->second.ok; + if (it != g_moe_cat.end()) return it->second.ok && it->second.w1.bits == 4u; } - MoECat4 cat; + MoECat cat; cat.ok = moe_build_set4(cat.w1, w1s, num_experts) && moe_build_set4(cat.w3, w3s, num_experts) && moe_build_set4(cat.w2, w2s, num_experts) && cat.w1.K == cat.w3.K && cat.w1.N == cat.w3.N && cat.w2.K >= cat.w1.N; - if (!cat.ok) cat = MoECat4{}; + if (!cat.ok) cat = MoECat{}; std::lock_guard lk(g_resident_mu); auto ins = g_moe_cat.emplace(key, cat); return ins.first->second.ok; } -bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void* probs, - const void* topk, const CactusQuantMatrix* w1_0, - uint32_t num_experts, uint32_t top_k, uint32_t tokens, - uint32_t act, uint32_t normalize, float eps, float scaling) { - if (!ctx().ok || !ctx().psoMoeT || top_k == 0 || top_k > 16 || tokens == 0) return false; - g_resident_mu.lock(); - auto it = g_moe_cat.find(resident_key(w1_0)); - bool hit = it != g_moe_cat.end() && it->second.ok; - g_resident_mu.unlock(); - if (!hit) return false; - MoECat4& C = it->second; +bool cactus_metal_moe_cq2_build(const CactusQuantMatrix* w1s, const CactusQuantMatrix* w3s, + const CactusQuantMatrix* w2s, uint32_t num_experts) { + if (!ctx().ok || !ctx().psoMoeTLow || !ctx().psoMoeT2Low || !ctx().psoMoeUpLow || !ctx().psoMoeDownLow) + return false; + const uint64_t key = resident_key(&w1s[0]); + { + std::lock_guard lk(g_resident_mu); + auto it = g_moe_cat.find(key); + if (it != g_moe_cat.end()) return it->second.ok && it->second.w1.bits == 2u; + } + MoECat cat; + cat.ok = moe_build_set2(cat.w1, w1s, num_experts) + && moe_build_set2(cat.w3, w3s, num_experts) + && moe_build_set2(cat.w2, w2s, num_experts) + && cat.w1.K == cat.w3.K && cat.w1.N == cat.w3.N && cat.w2.K >= cat.w1.N; + if (!cat.ok) cat = MoECat{}; + std::lock_guard lk(g_resident_mu); + auto ins = g_moe_cat.emplace(key, cat); + return ins.first->second.ok && ins.first->second.w1.bits == 2u; +} + +static bool cactus_metal_encode_moe_gated_impl(void* out, const void* hidden, const void* probs, + const void* topk, const CactusQuantMatrix* w1_0, + uint32_t num_experts, uint32_t top_k, uint32_t tokens, + uint32_t act, uint32_t normalize, float eps, float scaling, + uint32_t bits) { + id pso_t = bits == 2u ? ctx().psoMoeTLow : ctx().psoMoeT; + id pso_t2 = bits == 2u ? ctx().psoMoeT2Low : ctx().psoMoeT2; + id pso_up = bits == 2u ? ctx().psoMoeUpLow : ctx().psoMoeUp; + id pso_down = bits == 2u ? ctx().psoMoeDownLow : ctx().psoMoeDownAcc; + if (!ctx().ok || !pso_t || !pso_t2 || !pso_up || !pso_down || top_k == 0 || top_k > 16 || tokens == 0) return false; + MoECat C; + { + std::lock_guard lk(g_resident_mu); + auto it = g_moe_cat.find(resident_key(w1_0)); + if (it == g_moe_cat.end() || !it->second.ok) return false; + C = it->second; + } + if (C.w1.bits != bits || C.w3.bits != bits || C.w2.bits != bits) return false; const uint32_t K1=C.w1.K, N1=C.w1.N, K2=C.w2.K, N2=C.w2.N; const size_t slots = (size_t)tokens*top_k; ensureEncoder(); @@ -1056,7 +1114,7 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void id code2 = recycled(slots*K2*2); { - [g_enc setComputePipelineState:ctx().psoMoeT2]; + [g_enc setComputePipelineState:pso_t2]; setBufAt(hidden, (size_t)tokens*K1*2, 0); setBufAt(topk, slots*sizeof(float), 1); [g_enc setBuffer:C.w1.rc offset:C.w1.rc_off atIndex:2]; @@ -1073,12 +1131,12 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void [g_enc setBytes:&Kc length:4 atIndex:12]; [g_enc setBytes:&tk length:4 atIndex:13]; [g_enc setBytes:&C.w1.stride length:4 atIndex:14]; [g_enc setBytes:&C.w3.stride length:4 atIndex:15]; - [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:128*(bits == 2u ? sizeof(__fp16) : sizeof(float)) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(K1/128u, 2u*top_k, tokens) - threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + threadsPerThreadgroup:MTLSizeMake(bits == 2u ? 128u : 32u,1,1)]; } { - [g_enc setComputePipelineState:ctx().psoMoeUp]; + [g_enc setComputePipelineState:pso_up]; [g_enc setBuffer:code1 offset:0 atIndex:0]; [g_enc setBuffer:code3 offset:0 atIndex:1]; setBufAt(topk, slots*sizeof(float), 2); [g_enc setBuffer:C.w1.pk offset:C.w1.pk_off atIndex:3]; [g_enc setBuffer:C.w1.nm offset:C.w1.nm_off atIndex:4]; [g_enc setBuffer:C.w1.cb offset:C.w1.cb_off atIndex:5]; @@ -1093,7 +1151,7 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void threadsPerThreadgroup:MTLSizeMake(256,1,1)]; } { - [g_enc setComputePipelineState:ctx().psoMoeT]; + [g_enc setComputePipelineState:pso_t]; [g_enc setBuffer:gu offset:0 atIndex:0]; setBufAt(topk, slots*sizeof(float), 1); [g_enc setBuffer:C.w2.rc offset:C.w2.rc_off atIndex:2]; @@ -1106,12 +1164,12 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void [g_enc setBytes:&xz length:4 atIndex:9]; [g_enc setBytes:&xy length:4 atIndex:10]; [g_enc setBytes:&tk length:4 atIndex:11]; [g_enc setBytes:&C.w2.stride length:4 atIndex:12]; - [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:128*(bits == 2u ? sizeof(__fp16) : sizeof(float)) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(K2/128u, top_k, tokens) - threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + threadsPerThreadgroup:MTLSizeMake(bits == 2u ? 128u : 32u,1,1)]; } { - [g_enc setComputePipelineState:ctx().psoMoeDownAcc]; + [g_enc setComputePipelineState:pso_down]; [g_enc setBuffer:code2 offset:0 atIndex:0]; setBufAt(topk, slots*sizeof(float), 1); setBufAt(probs, (size_t)tokens*num_experts*2, 2); @@ -1123,13 +1181,29 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void [g_enc setBytes:&eps length:4 atIndex:11]; [g_enc setBytes:&scaling length:4 atIndex:12]; [g_enc setBytes:&Ec length:4 atIndex:13]; [g_enc setBytes:&C.w2.stride length:4 atIndex:14]; - [g_enc setThreadgroupMemoryLength:top_k*16*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:top_k*(bits == 2u ? 4u : 16u)*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake((N2+15u)/16u, 1, tokens) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; } return true; } +bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void* probs, + const void* topk, const CactusQuantMatrix* w1_0, + uint32_t num_experts, uint32_t top_k, uint32_t tokens, + uint32_t act, uint32_t normalize, float eps, float scaling) { + return cactus_metal_encode_moe_gated_impl(out, hidden, probs, topk, w1_0, num_experts, + top_k, tokens, act, normalize, eps, scaling, 4u); +} + +bool cactus_metal_encode_moe_gated_cq2(void* out, const void* hidden, const void* probs, + const void* topk, const CactusQuantMatrix* w1_0, + uint32_t num_experts, uint32_t top_k, uint32_t tokens, + uint32_t act, uint32_t normalize, float eps, float scaling) { + return cactus_metal_encode_moe_gated_impl(out, hidden, probs, topk, w1_0, num_experts, + top_k, tokens, act, normalize, eps, scaling, 2u); +} + bool cactus_metal_encode_deltanet_decode(void* out, const void* q, const void* k, const void* v, const void* g, const void* b, const void* s, uint32_t B, uint32_t Hq, uint32_t Hv, @@ -1283,6 +1357,39 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; const uint32_t bits=W->bits, pgb=(gs*bits+7u)/8u; + if (bits == 2u && quant_lowbit_eligible(W) && ctx().psoT2f16 && ctx().psoActQ && ctx().psoG2mi8) { + ResW& rw = resident(W); + if (!rw.ok || !rw.cb_i8) return false; + size_t code_bytes = (size_t)M*K*sizeof(__fp16); + if (!g_code_buf_m || (size_t)g_code_buf_m.length < code_bytes) + g_code_buf_m = owned_shared(code_bytes); + if (!g_code_buf_m) return false; + ensureEncoder(); + + [g_enc setComputePipelineState:ctx().psoT2f16]; + setBufAt(lhs, (size_t)M*K*2, 0); [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:1]; + [g_enc setBuffer:rw.lsign offset:rw.ls_off atIndex:2]; [g_enc setBuffer:rw.rsign offset:rw.rs_off atIndex:3]; + [g_enc setBuffer:rw.perm offset:rw.pm_off atIndex:4]; [g_enc setBuffer:g_code_buf_m offset:0 atIndex:5]; + [g_enc setBytes:&K length:4 atIndex:6]; + [g_enc setThreadgroupMemoryLength:128u*sizeof(__fp16) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,M,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + + id actq = recycled((size_t)M*K); + id ascl = recycled((size_t)M*ng*sizeof(float)); + [g_enc setComputePipelineState:ctx().psoActQ]; + [g_enc setBuffer:g_code_buf_m offset:0 atIndex:0]; [g_enc setBuffer:actq offset:0 atIndex:1]; + [g_enc setBuffer:ascl offset:0 atIndex:2]; [g_enc setBytes:&gs length:4 atIndex:3]; + [g_enc dispatchThreadgroups:MTLSizeMake((size_t)M*ng,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + + [g_enc setComputePipelineState:ctx().psoG2mi8]; + [g_enc setBuffer:actq offset:0 atIndex:0]; [g_enc setBuffer:ascl offset:0 atIndex:1]; + [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:2]; [g_enc setBuffer:rw.cb_i8 offset:0 atIndex:3]; + [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:4]; setBufAt(out, (size_t)M*N*2, 5); + [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&N length:4 atIndex:7]; + [g_enc setBytes:&rw.cb_scale length:4 atIndex:8]; + [g_enc dispatchThreadgroups:MTLSizeMake((N+7u)/8u,M,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; + } const bool fast = ctx().ok && quant_fast_eligible(W); if (!fast || !ctx().psoGmma) return false; ResW& rw = resident(W); diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 9ecafc05b..e004fa17c 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -133,4 +133,9 @@ bool cactus_metal_moe_cq4_build(const CactusQuantMatrix*, const CactusQuantMatri const CactusQuantMatrix*, uint32_t) { return false; } bool cactus_metal_encode_moe_gated_cq4(void*, const void*, const void*, const void*, const CactusQuantMatrix*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, float) { return false; } +bool cactus_metal_moe_cq2_ready(const CactusQuantMatrix*) { return false; } +bool cactus_metal_moe_cq2_build(const CactusQuantMatrix*, const CactusQuantMatrix*, + const CactusQuantMatrix*, uint32_t) { return false; } +bool cactus_metal_encode_moe_gated_cq2(void*, const void*, const void*, const void*, const CactusQuantMatrix*, + uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, float) { return false; } #endif