From dfa3fb5061ad5e1274042cc86789365d310ebf8f Mon Sep 17 00:00:00 2001 From: jakmro Date: Mon, 29 Jun 2026 17:06:22 +0200 Subject: [PATCH 01/36] gemma-metal Signed-off-by: jakmro --- cactus-engine/CMakeLists.txt | 4 +- cactus-engine/cactus_engine.h | 6 + cactus-engine/src/complete.cpp | 2 + cactus-engine/src/init.cpp | 3 + cactus-engine/src/model.cpp | 35 +- cactus-engine/tests/run.cpp | 16 +- cactus-graph/CMakeLists.txt | 1 + cactus-graph/cactus_graph.h | 27 +- cactus-graph/src/execute.cpp | 627 ++++++++++- cactus-graph/src/gpu_fused.cpp | 325 ++++++ cactus-graph/src/ops_cache.cpp | 30 +- cactus-graph/src/ops_math.cpp | 15 +- cactus-graph/src/ops_nn.cpp | 12 +- cactus-graph/src/ops_tensor.cpp | 8 +- cactus-graph/src/param_io.cpp | 2 +- cactus-kernels/CMakeLists.txt | 17 + cactus-kernels/cmake/embed_msl.cmake | 8 + cactus-kernels/metal_backend.h | 104 ++ cactus-kernels/src/cactus_kernels.metal | 1191 +++++++++++++++++++++ cactus-kernels/src/metal_backend.mm | 1017 ++++++++++++++++++ cactus-kernels/src/metal_backend_stub.cpp | 58 + python/cactus/cli/__init__.py | 11 + python/cactus/cli/code.py | 3 +- python/cactus/cli/common.py | 12 + python/cactus/cli/compile.py | 1 + python/cactus/cli/run.py | 1 + 26 files changed, 3507 insertions(+), 29 deletions(-) create mode 100644 cactus-graph/src/gpu_fused.cpp create mode 100644 cactus-kernels/cmake/embed_msl.cmake create mode 100644 cactus-kernels/metal_backend.h create mode 100644 cactus-kernels/src/cactus_kernels.metal create mode 100644 cactus-kernels/src/metal_backend.mm create mode 100644 cactus-kernels/src/metal_backend_stub.cpp diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index eee348095..b1b2a81d3 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -56,7 +56,7 @@ target_link_libraries(cactus_engine PUBLIC cactus_graph) if(APPLE) target_link_libraries(cactus_engine PUBLIC - "-framework CoreML" "-framework Foundation" + "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" ) @@ -156,7 +156,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) target_link_libraries(cactus_engine_shared PRIVATE "${_curl}") endif() target_link_libraries(cactus_engine_shared PRIVATE - "-framework Accelerate" "-framework CoreML" "-framework Foundation" + "-framework Accelerate" "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" ) else() diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index 376628563..e3f40c5b2 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -34,6 +34,12 @@ CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); +// Backend selection. "metal" runs the Apple GPU / Metal fused path (when available), "cpu" the CPU path. +// Overrides the CACTUS_GPU[_FUSED] env default. Call before cactus_init / the first forward. +CACTUS_FFI_EXPORT void cactus_set_backend(const char* backend); +CACTUS_FFI_EXPORT const char* cactus_get_backend(void); // resolved: "metal" or "cpu" +CACTUS_FFI_EXPORT bool cactus_metal_available(void); // true if a Metal device is present + CACTUS_FFI_EXPORT int cactus_complete( cactus_model_t model, const char* messages_json, diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 51ef39b06..c5e732f25 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -816,6 +816,7 @@ int cactus_complete( CACTUS_LOG_DEBUG("complete", "Prompt tokens: " << prompt.tokens.size() << ", max_tokens: " << prompt.options.max_tokens); + g_cactus_force_cpu = false; bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); @@ -1428,6 +1429,7 @@ int cactus_benchmark_tokens( peak_ram_usage_mb = std::max(peak_ram_usage_mb, get_ram_usage_mb()); }; handle->model->reset_cache(); + g_cactus_force_cpu = false; auto prefill_start_time = std::chrono::high_resolution_clock::now(); size_t cache_prime_tokens = 0; uint32_t next_token = 0; diff --git a/cactus-engine/src/init.cpp b/cactus-engine/src/init.cpp index 7672d88f2..b5f2f55e9 100644 --- a/cactus-engine/src/init.cpp +++ b/cactus-engine/src/init.cpp @@ -349,6 +349,9 @@ bool matches_stop_sequence(const std::vector& generated_tokens, extern "C" { +void cactus_set_backend(const char* backend) { cactus_backend_select(backend); } +const char* cactus_get_backend(void) { return cactus_backend_name(); } + cactus_model_t cactus_init(const char* model_path, const char* corpus_dir, bool cache_index) { constexpr size_t DEFAULT_CONTEXT_SIZE = 512; // matches default sliding window size diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 412ccc2b2..327ccdf97 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -562,6 +562,8 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, } else if (decode_route_ != DecodeRoute::ENCODER_CROSS_KV_STEP && !components_.count("audio_encoder")) { CACTUS_LOG_WARN("model", "Bundle has no decoder_prefill_chunk component; prompts will prefill token-by-token (prefill speed ~= decode speed)."); } + if (decoder_ && decoder_->graph) decoder_->graph->prewarm_gpu_quant_weights(); + if (decoder_prefill_ && decoder_prefill_->graph) decoder_prefill_->graph->prewarm_gpu_quant_weights(); if (components_.count("decoder_embed_chunk") && components_.at("decoder_embed_chunk").graph) { decoder_embed_ = &components_.at("decoder_embed_chunk"); } @@ -881,13 +883,27 @@ void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { if (decode_route_ == DecodeRoute::DIRECT_DECODER_STEP) { write_int_input(*decoder_, "input_ids", static_cast(token_id)); write_int_input(*decoder_, "position_ids", static_cast(position)); - decoder_->graph->execute(); + const bool fused = cactus_backend_fused(); + if (!fused || !decoder_->graph->execute_gpu_fused()) decoder_->graph->execute(); maybe_capture_handoff_probe_hidden(*decoder_); return; } + const bool fused = cactus_backend_fused(); + static const bool nofold = std::getenv("CACTUS_FUSED_NOFOLD") != nullptr; + if (fused && !nofold) { + static int state = 0; static FusedEmbedCtx ctx; // 0=unknown 1=ok 2=unsupported + if (state == 0) state = encoder_->graph->extract_ple_pathway(ctx) ? 1 : 2; + if (state == 1) { + ctx.token_id = static_cast(token_id); ctx.position = static_cast(position); + cactus_graph_set_fused_embed(&ctx); + bool ok = decoder_->graph->execute_gpu_fused(); + cactus_graph_set_fused_embed(nullptr); + if (ok) { maybe_capture_handoff_probe_hidden(*decoder_); return; } + } + } run_encoder_step(token_id, position); copy_component_outputs_to_inputs(*encoder_, *decoder_); - decoder_->graph->execute(); + if (!fused || !decoder_->graph->execute_gpu_fused()) decoder_->graph->execute(); maybe_capture_handoff_probe_hidden(*decoder_); } @@ -1905,6 +1921,21 @@ uint32_t Model::argmax_logits_at(const BufferDesc& desc, void* ptr, size_t row_o float best_v = -std::numeric_limits::infinity(); float second_v = -std::numeric_limits::infinity(); const auto& tool_bias = tool_constrainer_.get_bias(); + { + uint32_t gidx; float gbest, gsecond; + if (tool_bias.empty() && vocab_bias_.empty() && suppressed_token_id_ < 0 && + cactus_graph_gpu_argmax(&gidx, &gbest, &gsecond)) { + if (out_uncertainty) { + float confidence = 1.0f; + if (std::isfinite(gbest) && std::isfinite(gsecond)) { + float margin = std::max(-60.0f, std::min(60.0f, gbest - gsecond)); + confidence = 1.0f / (1.0f + std::exp(-margin)); + } + *out_uncertainty = std::max(0.0f, std::min(1.0f, 1.0f - confidence)); + } + return gidx; + } + } auto score_with_bias = [&](size_t token_id, float value) { auto tool_it = tool_bias.find(static_cast(token_id)); if (tool_it != tool_bias.end()) value += tool_it->second; diff --git a/cactus-engine/tests/run.cpp b/cactus-engine/tests/run.cpp index 359e4bd3c..f07e1bde6 100644 --- a/cactus-engine/tests/run.cpp +++ b/cactus-engine/tests/run.cpp @@ -742,7 +742,7 @@ std::string build_messages(const std::string& system_prompt, void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 - << " [--system ] [--image ] [--audio ]" + << " [--backend cpu|metal] [--system ] [--image ] [--audio ]" << " [--prompt ] [--input-ids ] [--input-ids-file ] [--max-new-tokens ]" << " [--result-json ] [--thinking] [--no-cloud-handoff]" << " [--confidence-threshold ] [--cloud-timeout-ms ] [-h|--help]\n"; @@ -772,6 +772,7 @@ int main(int argc, char** argv) { bool auto_handoff = true; double confidence_threshold = -1.0; int cloud_timeout_ms = 15000; + std::string backend; int i = 2; auto need_value = [&](const char* flag) -> bool { @@ -821,6 +822,9 @@ int main(int argc, char** argv) { } else if (arg == "--cloud-timeout-ms") { if (!need_value("--cloud-timeout-ms")) return 1; cloud_timeout_ms = std::max(0, std::atoi(argv[++i])); + } else if (arg == "--backend") { + if (!need_value("--backend")) return 1; + backend = argv[++i]; } else { std::cerr << "Error: unknown option '" << arg << "'\n"; print_usage(argv[0]); @@ -851,6 +855,16 @@ int main(int argc, char** argv) { } } + // --backend metal enables the fused Metal GPU path, gated by CACTUS_GPU_FUSED in cactus-graph + // (read once at the first forward, so it must be set before cactus_init). Default is CPU. + if (backend == "metal" || backend == "gpu") { + setenv("CACTUS_GPU_FUSED", "1", 1); + std::cout << "Backend: Metal GPU\n"; + } else if (!backend.empty() && backend != "cpu") { + std::cerr << "Error: unknown backend '" << backend << "' (expected 'cpu' or 'metal')\n"; + return 1; + } + cactus_log_set_callback(log_callback, nullptr); std::cout << "Loading model from " << model_path << "...\n"; diff --git a/cactus-graph/CMakeLists.txt b/cactus-graph/CMakeLists.txt index c82b62a02..6f2704ce8 100644 --- a/cactus-graph/CMakeLists.txt +++ b/cactus-graph/CMakeLists.txt @@ -12,6 +12,7 @@ add_library(cactus_graph STATIC src/core.cpp src/builder.cpp src/execute.cpp + src/gpu_fused.cpp src/io.cpp src/param_io.cpp src/ops_math.cpp diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index b14f824d4..4641e3359 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -17,6 +17,13 @@ #include #include +// Backend selection: "metal" or "cpu". Overrides the CACTUS_GPU[_FUSED] env default. +// cactus_backend_metal() gates the GPU op path; cactus_backend_fused() gates the fully-fused decode. +void cactus_backend_select(const char* backend); +bool cactus_backend_metal(); +bool cactus_backend_fused(); +const char* cactus_backend_name(); + namespace cactus { enum class LogLevel { @@ -81,7 +88,7 @@ class Logger { #define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) #define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) -enum class ComputeBackend { CPU, NPU }; +enum class ComputeBackend { CPU, NPU, GPU }; enum class Activation { SILU, GELU, GELU_ERF, RELU, SIGMOID, TANH }; @@ -452,6 +459,20 @@ namespace GraphFile { struct SerializedGraph; } +struct FusedEmbedCtx { + int token_id = -1, position = 0; + CactusQuantMatrix ple{}; + CactusQuantMatrix proj{}; + const void* rms_weight = nullptr; + float emb_scale = 0, ple_scale = 0, proj_scale = 0, final_scale = 0, rms_eps = 1e-6f; + bool ok = false; +}; +void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx); + +bool cactus_graph_gpu_argmax(uint32_t* idx, float* best, float* second); + +extern bool g_cactus_force_cpu; + class CactusGraph { public: CactusGraph(); @@ -705,6 +726,8 @@ class CactusGraph { void invalidate_persistent(size_t persistent_node_id); void execute(const std::string& profile_file = ""); + bool execute_gpu_fused(); + bool extract_ple_pathway(FusedEmbedCtx& ctx) const; void hard_reset(); void soft_reset(); void soft_reset_keep_pool(); @@ -730,6 +753,7 @@ class CactusGraph { const std::vector& backup); void allocate_buffers(); size_t get_node_count() const; + void prewarm_gpu_quant_weights(); void set_runtime_input_shape(size_t node_id, const std::vector& shape); void set_input_dynamic_dims(size_t node_id, const std::vector& dynamic_dims); bool has_dynamic_shapes() const { return has_dynamic_shapes_; } @@ -756,6 +780,7 @@ class CactusGraph { std::unordered_set populated_node_ids_; std::unordered_set embedded_input_node_ids_; std::unordered_set retained_output_node_ids_; + std::unordered_map> gpu_persistent_acts_; }; namespace GraphFile { diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index fb1481cbb..d3a7b73a9 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -1,5 +1,6 @@ #include "../cactus_graph.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include #include #include @@ -9,10 +10,29 @@ #include #include #include +#include #include #include #include +static int g_backend_override = -1; // -1 = env default, 0 = cpu, 1 = metal +void cactus_backend_select(const char* backend) { + if (!backend) { g_backend_override = -1; return; } + if (std::strcmp(backend, "metal") == 0) g_backend_override = 1; + else if (std::strcmp(backend, "cpu") == 0) g_backend_override = 0; +} +bool cactus_backend_metal() { + if (g_backend_override == 0) return false; + if (g_backend_override == 1) return cactus_metal_available(); + return (std::getenv("CACTUS_GPU") != nullptr || std::getenv("CACTUS_GPU_FUSED") != nullptr) && cactus_metal_available(); +} +bool cactus_backend_fused() { + if (g_backend_override == 0) return false; + if (g_backend_override == 1) return cactus_metal_available(); + return std::getenv("CACTUS_GPU_FUSED") != nullptr && cactus_metal_available(); +} +const char* cactus_backend_name() { return cactus_backend_metal() ? "metal" : "cpu"; } + using ComputeFn = void(*)(GraphNode&, const nodes_vector&, const node_index_map_t&); #define DECLARE_COMPUTE(name) \ @@ -63,6 +83,7 @@ DECLARE_COMPUTE(compute_slice_node); DECLARE_COMPUTE(compute_embedding_node); DECLARE_COMPUTE(compute_concat_node); DECLARE_COMPUTE(compute_cat_node); +bool cactus_kv_cache_grow(BufferDesc&, size_t, size_t); DECLARE_COMPUTE(compute_index_node); DECLARE_COMPUTE(compute_bilinear_interpolation_node); DECLARE_COMPUTE(compute_sample_node); @@ -429,10 +450,425 @@ void CactusGraph::infer_shapes() { runtime_shapes_dirty_ = false; } +static bool gpu_op_enabled(const char* name) { + static const std::string spec = []{ const char* e=std::getenv("CACTUS_GPU_OPS"); return e?std::string(e):std::string(); }(); + if (spec.empty()) return true; + return spec.find(name) != std::string::npos; +} + +static void row_strides(const std::vector& shape, size_t* out) { + size_t s=1; for (int k=(int)shape.size()-1; k>=0; --k){ out[k]=s; s*=shape[k]; } +} +static void bcast_strides(const std::vector& in_shape, const std::vector& out_shape, uint32_t* out) { + size_t off = out_shape.size() - in_shape.size(); + size_t istr[8]; row_strides(in_shape, istr); + for (size_t d=0; d 0)) return false; + if (!fp16(lhs)) return false; + CactusQuantMatrix mat = rhs.to_cq_matrix(); + if (M == 1) return cactus_metal_encode_quant_matmul(out.get_data(), lhs.get_data(), &mat); + return cactus_metal_encode_quant_matmul_m(out.get_data(), lhs.get_data(), &mat, (uint32_t)M); + } + case OpType::ADD: case OpType::ADD_CLIPPED: case OpType::SUBTRACT: + case OpType::MULTIPLY: case OpType::DIVIDE: { + const auto& a = get_input(node, 0, nodes, map); + const auto& b = get_input(node, 1, nodes, map); + if (!fp16(a) || !fp16(b) || !fp16(out)) return false; + int code = node.op_type==OpType::ADD?0: node.op_type==OpType::ADD_CLIPPED?1: + node.op_type==OpType::SUBTRACT?2: node.op_type==OpType::MULTIPLY?3:4; + if (a.total_size == out.total_size && b.total_size == out.total_size) + return cactus_metal_encode_binary(code, out.get_data(), a.get_data(), b.get_data(), out.total_size); + const auto& osh = out.shape; + uint32_t nd = (uint32_t)osh.size(); + if (nd == 0 || nd > 8 || a.shape.size() > nd || b.shape.size() > nd) return false; + uint32_t oshape[8], astr[8], bstr[8]; + for (uint32_t d=0; d 8 || in.shape.size() != nd || out.shape.size() != nd) return false; + size_t istr[8]; row_strides(in.shape, istr); + uint32_t oshape[8], sstride[8]; + for (uint32_t d=0; d 8 || axis >= nd || out.shape.size() != nd) return false; + size_t istr[8]; row_strides(in.shape, istr); + uint32_t oshape[8], sstride[8]; + for (uint32_t d=0; d= in.shape.size()) return false; + size_t istr[8]; row_strides(in.shape, istr); + size_t slice = istr[axis], block = istr[axis-1]; + uint32_t oshape[2] = { (uint32_t)(in.total_size/block), (uint32_t)slice }; + uint32_t sstride[2] = { (uint32_t)block, 1u }; + return cactus_metal_encode_strided_copy(out.get_data(), in.get_data(), oshape, sstride, 2, + (uint32_t)out.total_size, (uint32_t)(node.params.index_value * slice), in.byte_size, out.byte_size); + } + case OpType::CAT: { + if (node.input_ids.size() < 2 || out.shape.empty()) return false; + size_t axis = (size_t)node.params.axis; + uint32_t nd = (uint32_t)out.shape.size(); + if (axis >= nd || nd > 8) return false; + size_t ostr[8]; row_strides(out.shape, ostr); + size_t axis_off = 0; + for (size_t ii = 0; ii < node.input_ids.size(); ++ii) { + const auto& cin = get_input(node, ii, nodes, map); + if (!fp16(cin) || cin.shape.size() != nd) return false; + uint32_t ishape[8], ostride[8]; + for (uint32_t d=0; d(out.precision), + in.get_data(), static_cast(in.precision), in.total_size); + } + case OpType::ATTENTION_CACHED: { + if (node.input_ids.size() < 5) return false; + const auto& qb = get_input(node, 0, nodes, map); + const auto& knew = get_input(node, 1, nodes, map); + const auto& vnew = get_input(node, 2, nodes, map); + const auto& kcache = get_input(node, 3, nodes, map); + const auto& vcache = get_input(node, 4, nodes, map); + if (std::getenv("CACTUS_GPU_DEBUG")) { + static int dq=0; + if (qb.shape.size()<2 || qb.shape[1]==1) if (dq++ < 4) std::cerr << "[attn] q.shape.size=" << qb.shape.size() + << " dims=[" << (qb.shape.size()>0?qb.shape[0]:0) << "," << (qb.shape.size()>1?qb.shape[1]:0) + << "," << (qb.shape.size()>2?qb.shape[2]:0) << "] kc.prec=" << (int)kcache.precision << "\n"; + } + if (qb.shape.size() < 3) return false; + size_t batch=qb.shape[0], seq=qb.shape[1], nqh=qb.shape[2]; + if (batch != 1) return false; + if (kcache.precision == Precision::FP16 || vcache.precision == Precision::FP16) return false; + const uint64_t* km = reinterpret_cast(kcache.get_data()); + const uint64_t* vm = reinterpret_cast(vcache.get_data()); + if (!km || !vm) return false; + size_t cache_len=km[0], max_seq=km[1], kv_heads=km[2], hdim=km[3], v_max=vm[1]; + if (kv_heads == 0 || hdim == 0 || nqh % kv_heads != 0) return false; + size_t v_hdim = node.params.v_head_dim > 0 ? node.params.v_head_dim : hdim; + size_t new_seq_len = knew.total_size / (kv_heads * hdim); + size_t history_len = cache_len >= new_seq_len ? cache_len - new_seq_len : 0; + size_t po = node.params.position_offset, pos, new_len = seq; + if (po == std::numeric_limits::max()) pos = history_len; + else if (po == std::numeric_limits::max() - 1) { + pos = (cache_len >= seq) ? cache_len - seq : 0; history_len = cache_len; new_len = 0; + } else pos = po; + size_t total_keys = history_len + new_len; + size_t win = node.params.window_size; + size_t kv_start = (win > 0 && pos > win) ? pos - win : 0; + if (pos > history_len) kv_start = 0; + size_t kv_end = node.params.is_causal ? std::min(total_keys, pos + 1) : total_keys; + float scale = node.params.scale != 0.0f ? node.params.scale : 1.0f/std::sqrt((float)hdim); + const char* bk = static_cast(kcache.get_data()); + const char* bv = static_cast(vcache.get_data()); + size_t ngK=(hdim+31)/32, ngV=(v_hdim+31)/32; + if (seq > 1) { + size_t sink = km[4]; + uint32_t ringv = (win > 0 && max_seq > 2*sink + 1) ? (uint32_t)(max_seq - 2*sink - 1) : 0u; + return cactus_metal_encode_attention_i8_prefill( + out.get_data(), qb.get_data(), knew.get_data(), vnew.get_data(), + bk + 64, bv + 64, + bk + 64 + max_seq*kv_heads*hdim, bv + 64 + v_max*kv_heads*v_hdim, + (uint32_t)nqh, (uint32_t)kv_heads, (uint32_t)hdim, (uint32_t)v_hdim, + (uint32_t)history_len, (uint32_t)new_len, (uint32_t)pos, + (uint32_t)win, node.params.is_causal ? 1u : 0u, (uint32_t)seq, scale, + max_seq*kv_heads*hdim, v_max*kv_heads*v_hdim, + max_seq*kv_heads*ngK*sizeof(float), v_max*kv_heads*ngV*sizeof(float), + (uint32_t)sink, ringv); + } + bool ok = cactus_metal_encode_attention_i8( + out.get_data(), qb.get_data(), knew.get_data(), vnew.get_data(), + bk + 64, bv + 64, + bk + 64 + max_seq*kv_heads*hdim, bv + 64 + v_max*kv_heads*v_hdim, + (uint32_t)nqh, (uint32_t)kv_heads, (uint32_t)hdim, (uint32_t)v_hdim, + (uint32_t)history_len, (uint32_t)total_keys, (uint32_t)kv_start, (uint32_t)kv_end, scale, + history_len*kv_heads*hdim, history_len*kv_heads*v_hdim, + history_len*kv_heads*ngK*sizeof(float), history_len*kv_heads*ngV*sizeof(float)); + if (std::getenv("CACTUS_GPU_DEBUG")) { static int d2=0; if (d2++ < 3) + std::cerr << "[attn2] nqh="< ok="<(cache.get_data()); + size_t current_len=km[0], max_len=km[1], kv_heads=km[2], hdim=km[3], sink=km[4], num_slots=km[5]; + if (num_slots != 1 || kv_heads == 0 || hdim == 0) return false; + size_t new_seq_len = new_kv.total_size / (kv_heads * hdim); + if (new_seq_len > 1) { + size_t ceiling = cache_node.params.max_cache_seq_len, ws = node.params.window_size; + bool sliding = ws > 0 && ws < ceiling; + size_t new_total = current_len + new_seq_len; + char* base = static_cast(cache.get_data()); + size_t ngK = (hdim + 31)/32; + if (sliding) { + uint32_t W = (uint32_t)(max_len - sink - 1); + if (!cactus_metal_encode_kv_append_ring_i8_m(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)current_len, 32, (uint32_t)new_seq_len, (uint32_t)sink, W, + new_kv.byte_size, max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = new_total; + } else if (ws > 0 && new_total > max_len) { + size_t window = max_len; + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) return false; + size_t remaining = std::min(tail_capacity - new_seq_len, current_len - keep_sink); + size_t shift_src = current_len - remaining; + if (!cactus_metal_encode_kv_append_sliding_i8_m(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)keep_sink, (uint32_t)remaining, (uint32_t)shift_src, 32, (uint32_t)new_seq_len, + new_kv.byte_size, max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = keep_sink + remaining + new_seq_len; + } else { + if (new_total > max_len) { + if (!cactus_kv_cache_grow(cache, new_total, ceiling)) return false; + km = reinterpret_cast(cache.get_data()); + max_len = km[1]; base = static_cast(cache.get_data()); + if (new_total > max_len) return false; + } + if (!cactus_metal_encode_kv_append_i8_m(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)current_len, 32, (uint32_t)new_seq_len, + new_kv.byte_size, max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = new_total; + } + if (out.get_data()) *out.data_as() = static_cast(km[0]); + return true; + } + size_t ceiling = cache_node.params.max_cache_seq_len, ws = node.params.window_size; + bool sliding = ws > 0 && ws < ceiling; + size_t new_total = current_len + new_seq_len; + char* base = static_cast(cache.get_data()); + size_t ngK = (hdim + 31)/32; + if (!sliding && new_total > max_len) { + if (!cactus_kv_cache_grow(cache, new_total, ceiling)) return false; + km = reinterpret_cast(cache.get_data()); + max_len = km[1]; base = static_cast(cache.get_data()); + if (new_total > max_len) return false; + } + size_t window = sliding ? ws : max_len; + if (new_total > window) { + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) return false; + size_t remaining = std::min(tail_capacity - new_seq_len, current_len - keep_sink); + size_t shift_src = current_len - remaining; + if (!cactus_metal_encode_kv_append_sliding_i8(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)keep_sink, (uint32_t)remaining, (uint32_t)shift_src, 32, new_kv.byte_size, + max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = keep_sink + remaining + new_seq_len; + if (out.get_data()) *out.data_as() = static_cast(km[0]); + return true; + } + if (!cactus_metal_encode_kv_append_i8(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)current_len, 32, new_kv.byte_size, + max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = new_total; + if (out.get_data()) *out.data_as() = static_cast(new_total); + return true; + } + case OpType::EMBEDDING: { + if (node.input_ids.size() < 2) return false; + const auto& emb = get_input(node, 0, nodes, map); + const auto& idxb = get_input(node, 1, nodes, map); + if (!fp16(out)) return false; + size_t M = idxb.total_size; + if (M == 0 || M > 4096) return false; + std::vector rows(M); + if (idxb.precision == Precision::FP32) { const float* p = idxb.data_as(); for (size_t i=0;i(); for (size_t i=0;i(); for (size_t i=0;i 0) { + CactusQuantMatrix W = emb.to_cq_matrix(); + if (W.flags & CACTUS_QUANT_FLAG_ORTHOGONAL) + return cactus_metal_encode_embedding_ortho_m(out.get_data(), &W, rows.data(), (uint32_t)M); + return cactus_metal_encode_embedding_hadamard_m(out.get_data(), &W, rows.data(), (uint32_t)M); + } + if (emb.precision == Precision::FP16 && emb.shape.size() >= 2) { + uint32_t D = (uint32_t)emb.shape[1]; + return cactus_metal_encode_gather_f16(out.get_data(), emb.get_data(), + emb.total_size * sizeof(__fp16), rows.data(), (uint32_t)M, D); + } + return false; + } + default: return false; + } +} + +namespace { +struct CpuOpProf { double ns = 0.0; uint64_t inv = 0; }; +std::map> g_cpu_prof; +std::map g_cpu_prof_calls; +void cpu_prof_dump() { + for (auto& nc : g_cpu_prof) { + size_t nn = nc.first; uint64_t calls = g_cpu_prof_calls[nn]; + if (!calls) continue; + double tot = 0.0; for (auto& op : nc.second) tot += op.second.ns; + std::vector> v(nc.second.begin(), nc.second.end()); + std::sort(v.begin(), v.end(), [](const std::pair& a, const std::pair& b){ + return a.second.ns > b.second.ns; }); + std::cerr << "\n=== CPU op profile: graph n=" << nn << " | " << calls << " call(s) | " + << std::fixed << std::setprecision(3) << (tot/calls/1e6) << " ms/call total ===\n"; + std::cerr << " " << std::left << std::setw(24) << "op" << std::right + << std::setw(11) << "ms/call" << std::setw(11) << "cnt/call" << std::setw(9) << "share\n"; + for (auto& e : v) { + std::cerr << " " << std::left << std::setw(24) << get_op_name((OpType)e.first) << std::right + << std::fixed << std::setprecision(4) << std::setw(11) << (e.second.ns/calls/1e6) + << std::setprecision(1) << std::setw(11) << ((double)e.second.inv/calls) + << std::setw(8) << (100.0*e.second.ns/tot) << "%\n"; + } + } +} +inline void cpu_prof_begin(size_t n) { + static bool reg = false; + if (!reg) { std::atexit(cpu_prof_dump); reg = true; } + g_cpu_prof_calls[n]++; +} +} + void CactusGraph::execute(const std::string& profile_file) { BufferPool& pool = buffer_pool_; const size_t n = nodes_.size(); + static const bool gpu_timing = std::getenv("CACTUS_GPU_TIMING") != nullptr; + auto _tt = [](){ return std::chrono::high_resolution_clock::now(); }; + auto _t0 = _tt(); infer_shapes(); + auto _t_infer = _tt(); + + if (std::getenv("CACTUS_OP_HIST")) { + static std::unordered_set seen_sizes; + if (seen_sizes.insert(n).second) { + std::unordered_map hist; + for (size_t i = 0; i < n; ++i) hist[static_cast(nodes_[i]->op_type)]++; + std::cerr << "[ophist] graph nodes=" << n << "\n"; + for (auto& kv : hist) + std::cerr << " " << get_op_name(static_cast(kv.first)) + << " : " << kv.second << "\n"; + } + } + + if (std::getenv("CACTUS_GRAPH_DUMP")) { + static std::unordered_set dumped; + if (dumped.insert(n).second) { + auto shp=[](const BufferDesc&b){ std::string s="["; for(size_t d=0;dsecond:SIZE_MAX); if(j)std::cerr<<","; std::cerr<<(long)idx; } + std::cerr << "]"; + if (nd.params.axis) std::cerr << " axis=" << nd.params.axis; + if (nd.params.epsilon != 0.0f) std::cerr << " eps=" << nd.params.epsilon; + if (nd.params.scalar != 0.0f) std::cerr << " scalar=" << nd.params.scalar; + if (!nd.params.permutation.empty()){ std::cerr<<" perm=["; for(size_t d=0;d int { const char* val = std::getenv(name); @@ -502,15 +938,34 @@ void CactusGraph::execute(const std::string& profile_file) { return true; }; - auto may_alias_input = [](const GraphNode& node) { - return (node.op_type == OpType::SLICE && node.params.axis == 0) || - (node.op_type == OpType::INDEX && node.params.axis == 0); - }; - - auto preallocates_output = [](const GraphNode& node) { - return !((node.op_type == OpType::SLICE && node.params.axis == 0) || - (node.op_type == OpType::INDEX && node.params.axis == 0)); + const bool gpu_mode = cactus_backend_metal(); + const bool gpu_mode_req = gpu_mode && !g_cactus_force_cpu; + auto aliases_input = [&](const GraphNode& node) -> bool { + if (node.op_type == OpType::SLICE && !node.input_ids.empty()) { + auto it = node_index_map_.find(node.input_ids[0]); + if (it != node_index_map_.end()) { + const auto& ish = nodes_[it->second]->output_buffer.shape; + size_t ax = static_cast(node.params.axis); + if (ax < ish.size()) { + size_t outer = 1; + for (size_t d = 0; d < ax; ++d) outer *= ish[d]; + if (outer == 1) return true; + } + } + } + if (node.op_type == OpType::INDEX && node.params.axis == 0) return true; + if (!gpu_mode_req) return false; + if (node.op_type == OpType::VIEW || node.op_type == OpType::RESHAPE || node.op_type == OpType::FLATTEN) + return true; + if (node.op_type == OpType::PRECISION_CAST && !node.input_ids.empty()) { + auto it = node_index_map_.find(node.input_ids[0]); + if (it != node_index_map_.end() && + nodes_[it->second]->output_buffer.precision == node.output_buffer.precision) return true; + } + return false; }; + auto may_alias_input = aliases_input; + auto preallocates_output = [&](const GraphNode& node) { return !aliases_input(node); }; std::vector last_use(n, 0); std::vector use_count(n, 0); @@ -544,14 +999,145 @@ void CactusGraph::execute(const std::string& profile_file) { release_after[last_use[i]].push_back(i); } + static const bool gpu_verify = std::getenv("CACTUS_GPU_VERIFY") != nullptr; + if (gpu_mode_req && !need_debug) { + auto _t_live = _tt(); + cactus_metal_session_begin(); + cactus_metal_set_active(true); + std::unordered_map gpu_fallback_hist; + auto gpu_release = [&](size_t idx) { + GraphNode& nd = *nodes_[idx]; + if (aliases_input(nd)) nd.output_buffer.external_data = nullptr; + }; + const bool concurrent = cactus_metal_concurrent(); + struct Acc { uintptr_t lo, hi; bool w; }; + std::vector accessed; + auto conflicts = [&](uintptr_t lo, uintptr_t hi, bool w) { + for (auto& a : accessed) if ((w || a.w) && lo < a.hi && a.lo < hi) return true; + return false; + }; + auto hazard_barrier = [&](GraphNode& nd) { + if (!concurrent) return; + const bool allW = (nd.op_type == OpType::KV_CACHE_APPEND); + std::vector mine; bool hit = false; + for (size_t j = 0; j < nd.input_ids.size(); ++j) { + const auto& ib = get_input(nd, j, nodes_, node_index_map_); + if (!ib.get_data()) continue; + uintptr_t lo = (uintptr_t)ib.get_data(), hi = lo + ib.byte_size; + if (conflicts(lo, hi, allW)) hit = true; + mine.push_back({lo, hi, allW}); + } + if (nd.output_buffer.get_data()) { + uintptr_t lo = (uintptr_t)nd.output_buffer.get_data(), hi = lo + nd.output_buffer.byte_size; + if (conflicts(lo, hi, true)) hit = true; + mine.push_back({lo, hi, true}); + } + if (hit) { cactus_metal_barrier(); accessed.clear(); } + for (auto& m : mine) accessed.push_back(m); + }; + for (size_t i = 0; i < n; ++i) { + auto& node = nodes_[i]; + const OpType ot = node->op_type; + if (ot == OpType::INPUT) continue; + if (ot == OpType::KV_CACHE_STATE || ot == OpType::CONV_CACHE_STATE + || ot == OpType::RECURRENT_CACHE_STATE) { + dispatch_node(*node, nodes_, node_index_map_); + populated_node_ids_.insert(node->id); + for (size_t r : release_after[i]) gpu_release(r); + continue; + } + if (aliases_input(*node)) { + dispatch_node(*node, nodes_, node_index_map_); + for (size_t r : release_after[i]) gpu_release(r); + continue; + } + if (preallocates_output(*node)) { + size_t need = node->output_buffer.byte_size; + auto pit = gpu_persistent_acts_.find(node->id); + void* p = nullptr; + if (pit != gpu_persistent_acts_.end() && pit->second.second >= need) { + p = pit->second.first; + } else { + if (pit != gpu_persistent_acts_.end()) cactus_metal_free_shared(pit->second.first); + p = cactus_metal_alloc_shared(need); + if (p) gpu_persistent_acts_[node->id] = { p, need }; + } + if (p) { node->output_buffer.release_to_pool(pool); node->output_buffer.set_external(p); } + else node->output_buffer.resize_from_pool(pool); + } + hazard_barrier(*node); + bool encoded = try_encode_gpu(*node, nodes_, node_index_map_); + if (!encoded) { + gpu_fallback_hist[(int)ot]++; + cactus_metal_session_sync(); + accessed.clear(); + dispatch_node(*node, nodes_, node_index_map_); + } else if (gpu_verify) { + cactus_metal_session_sync(); + BufferDesc& ob = node->output_buffer; + std::vector gpu_copy(ob.byte_size); + std::memcpy(gpu_copy.data(), ob.get_data(), ob.byte_size); + dispatch_node(*node, nodes_, node_index_map_); + if (ob.precision == Precision::FP16) { + const __fp16* g = reinterpret_cast(gpu_copy.data()); + const __fp16* c = ob.data_as<__fp16>(); + double maxd=0; size_t bad=0; + for (size_t k=0;kmaxd)maxd=d; if(d>0.05)bad++; } + if (maxd > 0.05) { + static std::set seen; + std::string key = std::string(get_op_name(node->op_type)); + if (seen.insert(key).second) + std::cerr << "[gpu-verify] MISMATCH op=" << get_op_name(node->op_type) + << " id=" << node->id << " maxdiff=" << maxd << " bad=" << bad + << "/" << ob.total_size << " shape0=" << (ob.shape.empty()?0:ob.shape[0]) << "\n"; + } + } + } + if (ot == OpType::PERSISTENT) populated_node_ids_.insert(node->id); + for (size_t r : release_after[i]) gpu_release(r); + } + if (std::getenv("CACTUS_GPU_STATS") && n > 1000) { + static std::set seen; + if (seen.insert(n).second) { std::cerr << "[gpu-fallback] graph n=" << n << ":\n"; + for (auto& kv : gpu_fallback_hist) std::cerr << " " << get_op_name((OpType)kv.first) << " : " << kv.second << "\n"; } + } + cactus_metal_set_active(false); + auto _t_loop = _tt(); + cactus_metal_session_end(); + if (gpu_timing && n > 1000) { + using us = std::chrono::microseconds; + auto _t_done = _tt(); + static int _c = 0; static long _sum_total=0, _sum_loop=0; static int _cnt=0; + _sum_total += std::chrono::duration_cast(_t_done-_t0).count(); + _sum_loop += std::chrono::duration_cast(_t_loop-_t_live).count(); + _cnt++; + if (_c++ % 32 == 31) + std::cerr << "[gpu-timing] avg total execute=" << _sum_total/_cnt + << "us loop(encode+inner-sync)=" << _sum_loop/_cnt + << "us (CPU encode = total - on-GPU)\n"; + } + return; + } + if (!need_debug) { + static const bool cpu_prof = std::getenv("CACTUS_CPU_PROFILE") != nullptr; + if (cpu_prof) cpu_prof_begin(n); + auto run = [&](auto& nd) { + if (!cpu_prof) { dispatch_node(nd, nodes_, node_index_map_); return; } + auto t0 = std::chrono::high_resolution_clock::now(); + dispatch_node(nd, nodes_, node_index_map_); + auto& pr = g_cpu_prof[n][(int)nd.op_type]; + pr.ns += (double)std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - t0).count(); + pr.inv++; + }; for (size_t i = 0; i < n; ++i) { auto& node = nodes_[i]; if (node->op_type == OpType::INPUT) continue; if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE || node->op_type == OpType::RECURRENT_CACHE_STATE) { - dispatch_node(*node, nodes_, node_index_map_); + run(*node); populated_node_ids_.insert(node->id); for (size_t release_idx : release_after[i]) { nodes_[release_idx]->output_buffer.release_memory(pool); @@ -561,8 +1147,14 @@ void CactusGraph::execute(const std::string& profile_file) { if (preallocates_output(*node)) { node->output_buffer.resize_from_pool(pool); } - dispatch_node(*node, nodes_, node_index_map_); + run(*node); trace_nonfinite(i, *node); + if (n == 3861) { static const char* dn = std::getenv("CACTUS_DUMP_NODE"); + if (dn && i == (size_t)std::atoi(dn)) { static int dc=0; if (dc++<1) { + BufferDesc& b = node->output_buffer; const __fp16* p = b.data_as<__fp16>(); + if (p) std::cerr << "[dump] node " << i << " " << get_op_name(node->op_type) << ": " + << (float)p[0] << " " << (float)p[1] << " " << (float)p[2] << " " << (float)p[3] + << " | " << (float)p[4] << " " << (float)p[5] << "\n"; } } } if (node->op_type == OpType::PERSISTENT) { populated_node_ids_.insert(node->id); } @@ -1034,3 +1626,18 @@ void CactusGraph::soft_reset_keep_pool() { next_node_id_ = max_preserved_id + 1; debug_nodes_.clear(); } + +void CactusGraph::prewarm_gpu_quant_weights() { + const bool gpu = cactus_backend_metal() && std::getenv("CACTUS_NO_PREWARM") == nullptr; + if (!gpu) return; + for (auto& np : nodes_) { + GraphNode& node = *np; + if (node.op_type != OpType::MATMUL || node.input_ids.size() < 2) continue; + auto it = node_index_map_.find(node.input_ids[1]); + if (it == node_index_map_.end()) continue; + const BufferDesc& rhs = nodes_[it->second]->output_buffer; + if (!(PrecisionTraits::is_cq(rhs.precision) && rhs.group_size > 0)) continue; + CactusQuantMatrix mat = rhs.to_cq_matrix(); + cactus_metal_prewarm_quant(&mat); + } +} diff --git a/cactus-graph/src/gpu_fused.cpp b/cactus-graph/src/gpu_fused.cpp new file mode 100644 index 000000000..fa800738d --- /dev/null +++ b/cactus-graph/src/gpu_fused.cpp @@ -0,0 +1,325 @@ + + +#include "../cactus_graph.h" +#include "cactus_kernels.h" +#include "metal_backend.h" +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr int NL = 35, HID = 1536, NQH = 8, PLE_DIM = 256; +constexpr float RMS_EPS = 1e-6f, GATE_SCALE = 0.015625f, SOFTCAP = 30.0f; + +struct LayerW { + CactusQuantMatrix q, o, ple_down, ple_up, gate, up, down; + CactusQuantMatrix k, v; + const void *in_norm, *q_norm, *k_norm, *post_attn, *pre_ffn, *post_ffn, *post_ple, *scalar; + bool shared, global; + int hd, mlp; + void *kc, *vc; +}; + +struct Arena { + std::vector> bufs; + size_t cur = 0; + void reset() { cur = 0; } + void* fresh(size_t bytes) { + if (cur >= bufs.size()) { + void* p = cactus_metal_alloc_shared(bytes); + if (p) std::memset(p, 0, bytes); + bufs.push_back({p, bytes}); + } + return bufs[cur++].first; + } +}; + +struct GpuModel { + bool init = false; + LayerW L[NL]; + size_t kc_idx[15], vc_idx[15]; + const void *vnorm, *vnorm_g; + const void *cosL, *sinL, *cosG, *sinG; + const float *position; + const void *hidden; + const void *ple; + const void *final_norm; + CactusQuantMatrix lm_head; + void *cos_s, *sin_s, *cos_g, *sin_g; + void *argmax_buf = nullptr; + Arena arena; +}; +static GpuModel G; + +static bool g_gpu_argmax_valid = false; +static const float* g_gpu_argmax_buf = nullptr; + +} + +bool g_cactus_force_cpu = false; + +bool cactus_graph_gpu_argmax(uint32_t* idx, float* best, float* second) { + if (!g_gpu_argmax_valid || !g_gpu_argmax_buf) return false; + g_gpu_argmax_valid = false; + *best = g_gpu_argmax_buf[0]; *second = g_gpu_argmax_buf[1]; + *idx = (uint32_t)g_gpu_argmax_buf[2]; + return true; +} + +static FusedEmbedCtx g_fe; +void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx) { + if (ctx && ctx->ok) g_fe = *ctx; else g_fe.ok = false; +} + +bool CactusGraph::extract_ple_pathway(FusedEmbedCtx& ctx) const { + + if (nodes_.size() != 23) return false; + auto op = [&](size_t i){ return nodes_[i]->op_type; }; + if (op(3) != OpType::INPUT || op(4) != OpType::INPUT || op(5) != OpType::INPUT) return false; + if (op(8) != OpType::EMBEDDING || op(12) != OpType::MATMUL || op(19) != OpType::RMS_NORM) return false; + if (op(7) != OpType::SCALAR_MULTIPLY || op(9) != OpType::SCALAR_MULTIPLY || + op(14) != OpType::SCALAR_MULTIPLY || op(22) != OpType::SCALAR_MULTIPLY) return false; + const BufferDesc& pleW = nodes_[3]->output_buffer; + const BufferDesc& projW = nodes_[5]->output_buffer; + if (!pleW.is_cq() || (pleW.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) || !projW.is_cq()) return false; + ctx.ple = pleW.to_cq_matrix(); + ctx.proj = projW.to_cq_matrix(); + ctx.rms_weight = nodes_[4]->output_buffer.get_data(); + ctx.emb_scale = nodes_[7]->params.scalar; + ctx.ple_scale = nodes_[9]->params.scalar; + ctx.proj_scale = nodes_[14]->params.scalar; + ctx.final_scale = nodes_[22]->params.scalar; + ctx.rms_eps = nodes_[19]->params.epsilon; + if (!ctx.rms_weight || ctx.emb_scale == 0.0f || ctx.proj.N == 0) return false; + ctx.ok = true; + return true; +} + +bool CactusGraph::execute_gpu_fused() { + static const bool dbg = std::getenv("CACTUS_FUSED_DEBUG") != nullptr; + if (!cactus_metal_available()) return false; + if (g_cactus_force_cpu) return false; + const size_t n = nodes_.size(); + if (n < 3861) return false; + auto B = [&](size_t idx) -> BufferDesc& { return nodes_[idx]->output_buffer; }; + auto P = [&](size_t idx) -> void* { return B(idx).get_data(); }; + if (B(0).total_size != HID) return false; + if (nodes_[0]->op_type != OpType::INPUT || B(0).precision != Precision::FP16) return false; + g_gpu_argmax_valid = false; + + if (!G.init) { + + auto cq = [&](size_t idx){ return B(idx).to_cq_matrix(); }; + for (int l = 0; l < NL; ++l) { + LayerW& w = G.L[l]; + w.global = (l % 5 == 4); + w.hd = w.global ? 512 : 256; + w.shared = (l >= 15); + if (!w.shared) { + size_t b1 = 3 + 10*l, b2 = 334 + 6*l; + w.in_norm=P(b1); w.q_norm=P(b1+1); w.k_norm=P(b1+2); w.post_attn=P(b1+3); + w.pre_ffn=P(b1+4); w.gate=cq(b1+5); w.up=cq(b1+6); w.down=cq(b1+7); + w.post_ffn=P(b1+8); w.post_ple=P(b1+9); + w.q=cq(b2); w.k=cq(b2+1); w.v=cq(b2+2); w.o=cq(b2+3); w.ple_down=cq(b2+4); w.ple_up=cq(b2+5); + w.mlp = 6144; + } else { + int j = l - 15; size_t b1 = 153 + 9*j, b2 = 424 + 4*j; + w.in_norm=P(b1); w.q_norm=P(b1+1); w.post_attn=P(b1+2); w.pre_ffn=P(b1+3); + w.gate=cq(b1+4); w.up=cq(b1+5); w.down=cq(b1+6); w.post_ffn=P(b1+7); w.post_ple=P(b1+8); + w.q=cq(b2); w.o=cq(b2+1); w.ple_down=cq(b2+2); w.ple_up=cq(b2+3); + w.mlp = 12288; + } + w.scalar = P(505 + l); + } + G.vnorm = P(540); G.vnorm_g = P(541); + G.cosL = P(542); G.sinL = P(543); G.cosG = P(544); G.sinG = P(545); + G.final_norm = P(333); G.lm_head = B(504).to_cq_matrix(); + + std::vector cidx; + for (size_t i = 0; i < n && cidx.size() < 30; ++i) + if (nodes_[i]->op_type == OpType::KV_CACHE_STATE) cidx.push_back(i); + if (cidx.size() < 30) return false; + for (int l = 0; l < 15; ++l) { G.kc_idx[l] = cidx[2*l]; G.vc_idx[l] = cidx[2*l+1]; } + + auto sh = [&](size_t bytes){ void* p = cactus_metal_alloc_shared(bytes); if (p) std::memset(p,0,bytes); return p; }; + G.cos_s = sh(256*2); G.sin_s = sh(256*2); G.cos_g = sh(512*2); G.sin_g = sh(512*2); + G.argmax_buf = sh(3*sizeof(float)); + G.init = true; + } + + const bool fold = g_fe.ok; + + int pos; + if (fold) { + pos = g_fe.position; + } else { + G.hidden = P(0); G.ple = P(1); G.position = static_cast(P(2)); + if (!G.hidden || !G.ple || !G.position) return false; + pos = (int)G.position[0]; + } + for (int l = 0; l < 15; ++l) { G.L[l].kc = P(G.kc_idx[l]); G.L[l].vc = P(G.vc_idx[l]); } + for (int l = 15; l < NL; ++l) { int src = G.L[l].global ? 14 : 13; G.L[l].kc = G.L[src].kc; G.L[l].vc = G.L[src].vc; } + for (int l = 0; l < NL; ++l) if (!G.L[l].kc || !G.L[l].vc) return false; + + for (int l = 0; l < 15; ++l) { + const uint64_t* km = (const uint64_t*)G.L[l].kc; + if (G.L[l].global && km[0] + 1 > km[1]) { + static const bool dbg2 = (std::getenv("CACTUS_FUSED_TIMER") != nullptr); + if (dbg2) { static int n=0; if (n++<3) fprintf(stderr,"[fused-bail] L%d clen=%llu max=%llu hd=%d\n", + l,(unsigned long long)km[0],(unsigned long long)km[1],G.L[l].hd); } + return false; + } + } + + G.arena.reset(); + auto fresh = [&](size_t elems){ return G.arena.fresh(elems * 2); }; + cactus_metal_session_begin(); + auto g_t0 = std::chrono::high_resolution_clock::now(); + cactus_metal_set_active(true); + + void* h = fresh(HID); + const void* pleBase; + if (fold) { + auto bail = [&](){ cactus_metal_set_active(false); cactus_metal_session_end(); return false; }; + const uint32_t tok = (uint32_t)g_fe.token_id; + const int PK = (int)g_fe.proj.N; + + void* hr = fresh(HID); + if (!cactus_metal_encode_embedding_ortho(hr, tok, &G.lm_head)) return bail(); + cactus_metal_encode_scalar(2, h, hr, HID, g_fe.emb_scale); + + void* pe = fresh(g_fe.ple.K); + if (!cactus_metal_encode_embedding_hadamard(pe, tok, &g_fe.ple)) return bail(); + void* pa = fresh(PK); cactus_metal_encode_scalar(2, pa, pe, PK, g_fe.ple_scale); + void* pj = fresh(PK); + if (!cactus_metal_encode_quant_matmul(pj, h, &g_fe.proj)) return bail(); + void* pjs= fresh(PK); cactus_metal_encode_scalar(2, pjs, pj, PK, g_fe.proj_scale); + void* psum=fresh(PK); cactus_metal_encode_rms_norm_add(psum, pjs, g_fe.rms_weight, pa, NL, PLE_DIM, g_fe.rms_eps); + void* plt= fresh(PK); cactus_metal_encode_scalar(2, plt, psum, PK, g_fe.final_scale); + pleBase = plt; + } else { + std::memcpy(h, G.hidden, HID*2); + pleBase = G.ple; + } + if (dbg) { cactus_metal_session_sync(); const __fp16* hh=(const __fp16*)h; const __fp16* pp=(const __fp16*)pleBase; + fprintf(stderr,"[embed %s] tok=%d h[0..3]= %.5f %.5f %.5f %.5f | ple[0..3]= %.5f %.5f %.5f %.5f\n", + fold?"FOLD":"node", fold?g_fe.token_id:-1, (float)hh[0],(float)hh[1],(float)hh[2],(float)hh[3], + (float)pp[0],(float)pp[1],(float)pp[2],(float)pp[3]); } + + auto gather = [&](const void* tbl, void* dst, int hd){ std::memcpy(dst, (const char*)tbl + (size_t)pos*hd*2, hd*2); }; + gather(G.cosL, G.cos_s, 256); gather(G.sinL, G.sin_s, 256); + gather(G.cosG, G.cos_g, 512); gather(G.sinG, G.sin_g, 512); + + for (int l = 0; l < NL; ++l) { + + LayerW& w = G.L[l]; + const int hd = w.hd, QD = NQH*hd; + const void* rc = w.global ? G.cos_g : G.cos_s; + const void* rs = w.global ? G.sin_g : G.sin_s; + uint64_t* km = (uint64_t*)w.kc; uint64_t* vm = (uint64_t*)w.vc; + size_t mx = km[1], ng = (hd+31)/32; + + void* xn = fresh(HID); cactus_metal_encode_rms_norm(xn, h, w.in_norm, 1, HID, RMS_EPS); + void* q = fresh(QD); cactus_metal_encode_quant_matmul(q, xn, &w.q); + void* qn = fresh(QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, NQH, hd, RMS_EPS); + void* qr = fresh(QD); cactus_metal_encode_rope(qr, qn, rc, rs, NQH, hd); + + size_t hist, total; const void *knewp, *vnewp; size_t kv_start = 0, kv_end; + const uint32_t Wn = w.global ? 0u : (uint32_t)(km[1] - km[4] - 1); + const uint32_t Sn = w.global ? 0u : (uint32_t)km[4]; + const uint32_t Rn = (Wn > Sn) ? (Wn - Sn) : 1u; + if (!w.shared) { + void* k = fresh(hd); cactus_metal_encode_quant_matmul(k, xn, &w.k); + void* v = fresh(hd); cactus_metal_encode_quant_matmul(v, xn, &w.v); + void* kn = fresh(hd); cactus_metal_encode_rms_norm(kn, k, w.k_norm, 1, hd, RMS_EPS); + void* vn = fresh(hd); cactus_metal_encode_rms_norm(vn, v, w.global ? G.vnorm_g : G.vnorm, 1, hd, RMS_EPS); + void* kr = fresh(hd); cactus_metal_encode_rope(kr, kn, rc, rs, 1, hd); + size_t clen = km[0]; + size_t slot = (!w.global && clen >= (size_t)Wn) ? (size_t)(Sn + ((clen - Sn) % Rn)) : clen; + cactus_metal_encode_kv_append_i8(kr, (char*)w.kc+64, (char*)w.kc+64+mx*hd, + 1, hd, (uint32_t)slot, 32, hd*2, mx*hd, mx*ng*sizeof(float)); + cactus_metal_encode_kv_append_i8(vn, (char*)w.vc+64, (char*)w.vc+64+mx*hd, + 1, hd, (uint32_t)slot, 32, hd*2, mx*hd, mx*ng*sizeof(float)); + km[0] = clen + 1; vm[0] = clen + 1; + if (!w.global && clen >= (size_t)Wn) { + hist = Wn; total = Wn; knewp = nullptr; vnewp = nullptr; kv_end = Wn; + } else { + hist = clen; total = clen + 1; knewp = kr; vnewp = vn; + kv_end = std::min(total, (size_t)pos + 1); + } + } else { + size_t clen = km[0]; + if (!w.global && clen > (size_t)Wn) { + hist = Wn; total = Wn; knewp = nullptr; vnewp = nullptr; kv_end = Wn; + } else { + hist = clen; total = clen; knewp = qr; vnewp = qr; + kv_end = std::min(clen, (size_t)pos + 1); + } + } + void* attn = fresh(QD); + cactus_metal_encode_attention_i8(attn, qr, knewp, vnewp, + (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, + NQH, 1, hd, hd, (uint32_t)hist, (uint32_t)total, (uint32_t)kv_start, (uint32_t)kv_end, 1.0f, + hist*hd, hist*hd, hist*ng*sizeof(float), hist*ng*sizeof(float)); + void* o = fresh(HID); cactus_metal_encode_quant_matmul(o, attn, &w.o); + void* h1 = fresh(HID); cactus_metal_encode_rms_norm_add(h1, o, w.post_attn, h, 1, HID, RMS_EPS); h = h1; + + const int M = w.mlp; + void* xn2 = fresh(HID); cactus_metal_encode_rms_norm(xn2, h, w.pre_ffn, 1, HID, RMS_EPS); + void* gate= fresh(M); cactus_metal_encode_quant_matmul(gate, xn2, &w.gate); + void* up = fresh(M); cactus_metal_encode_quant_matmul(up, xn2, &w.up); + void* g3 = fresh(M); cactus_metal_encode_swiglu(g3, gate, up, M, GATE_SCALE); + void* mo = fresh(HID); cactus_metal_encode_quant_matmul(mo, g3, &w.down); + void* h2 = fresh(HID); cactus_metal_encode_rms_norm_add(h2, mo, w.post_ffn, h, 1, HID, RMS_EPS); h = h2; + + void* ps = fresh(PLE_DIM); cactus_metal_encode_quant_matmul(ps, h, &w.ple_down); + void* pm = fresh(PLE_DIM); cactus_metal_encode_swiglu(pm, ps, (const char*)pleBase + (size_t)l*PLE_DIM*2, PLE_DIM, 1.0f); + void* pu = fresh(HID); cactus_metal_encode_quant_matmul(pu, pm, &w.ple_up); + void* h3 = fresh(HID); cactus_metal_encode_rms_norm_add(h3, pu, w.post_ple, h, 1, HID, RMS_EPS); h = h3; + + float ls = (float)(*(const __fp16*)w.scalar); + void* h4 = fresh(HID); cactus_metal_encode_scalar(2, h4, h, HID, ls); h = h4; + + if (dbg) { cactus_metal_session_sync(); const __fp16* p=(const __fp16*)h; + fprintf(stderr,"[fused] L%d out: %.5f %.5f %.5f %.5f (ls=%.6f)\n",l,(float)p[0],(float)p[1],(float)p[2],(float)p[3],ls); } + } + + size_t V = B(3860).total_size; + void* fn = fresh(HID); cactus_metal_encode_rms_norm(fn, h, G.final_norm, 1, HID, RMS_EPS); + void* code = fresh(HID); + void* lg = fresh(V); + if (cactus_metal_encode_quant_matmul_ortho(lg, fn, code, &G.lm_head)) { + cactus_metal_encode_scalar(3, lg, lg, V, SOFTCAP); + cactus_metal_encode_unary(1, lg, lg, V); + cactus_metal_encode_scalar(2, lg, lg, V, SOFTCAP); + B(3860).set_external(lg); + if (G.argmax_buf && cactus_metal_encode_argmax(lg, (uint32_t)V, G.argmax_buf)) { + g_gpu_argmax_buf = (const float*)G.argmax_buf; + g_gpu_argmax_valid = true; + } + } else { + cactus_metal_session_sync(); + cactus_quant_matmul(&G.lm_head, (const __fp16*)fn, 1u, (__fp16*)lg); + __fp16* L=(__fp16*)lg; for (size_t i=0;i(t1 - g_t0).count(); + double syn = std::chrono::duration(t2 - t1).count(); + fprintf(stderr, "[fused-timer] pos=%d encode_cpu=%.2fms commit+wait=%.2fms total=%.2fms\n", + pos, enc, syn, enc+syn); + } else { + cactus_metal_session_end(); + } + return true; +} diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index fc2da274f..5e7f33097 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -1,5 +1,6 @@ #include "../cactus_graph.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include #include #include @@ -101,6 +102,10 @@ inline bool use_fp16_kv_cache() { constexpr size_t kInitialCacheEntries = 256; +inline bool kv_cache_resident() { + return cactus_backend_metal(); +} + inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { auto* meta = get_meta(buf); size_t cur = meta->max_seq_len; @@ -114,7 +119,17 @@ inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { size_t total = fp16_cache ? fp16_cache_elements(new_max, kv_heads, hdim) : cache_buffer_size(new_max, kv_heads, hdim); BufferDesc resized({total}, fp16_cache ? Precision::FP16 : Precision::INT8); - resized.allocate(); + // Keep the cache GPU-resident across grows: a plain CPU realloc would drop it out of the Metal + // shared map, forcing the fused/executor decode to snapshot the whole KV every token (O(context) + // CPU cost). Allocating shared keeps the live, offset-aware read path. CPU build: active=false. + bool shared = kv_cache_resident(); + void* old_data = buf.get_data(); + if (shared) { + void* p = cactus_metal_alloc_shared(resized.byte_size); + if (p) resized.set_external(p); else { resized.allocate(); shared = false; } + } else { + resized.allocate(); + } std::memset(resized.get_data(), 0, resized.byte_size); std::memcpy(resized.get_data(), buf.get_data(), sizeof(CacheMetadata)); @@ -130,6 +145,7 @@ inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { } get_meta(resized)->max_seq_len = new_max; buf = std::move(resized); + if (shared) cactus_metal_free_shared(old_data); // reclaim the old shared buffer (no-op if not shared) return true; } @@ -145,6 +161,10 @@ inline bool grow_cache_buffer(BufferDesc& buf, size_t needed, size_t ceiling) { } // namespace +bool cactus_kv_cache_grow(BufferDesc& buf, size_t needed, size_t ceiling) { + return grow_cache_buffer(buf, needed, ceiling); +} + void compute_kv_cache_state_node( GraphNode& node, const nodes_vector&, @@ -159,6 +179,7 @@ void compute_kv_cache_state_node( size_t max_seq; if (sliding) max_seq = std::min(ceiling, window + node.params.cache_sink_size + 1); else if (num_slots > 1) max_seq = ceiling; + else if (kv_cache_resident()) max_seq = std::min(ceiling, static_cast(4000)); else max_seq = std::min(ceiling, kInitialCacheEntries); size_t kv_heads = node.params.num_kv_heads; size_t hdim = node.params.head_dim; @@ -168,7 +189,12 @@ void compute_kv_cache_state_node( : cache_buffer_size(max_seq, kv_heads, hdim); node.output_buffer = BufferDesc({num_slots * per_slot}, fp16_cache ? Precision::FP16 : Precision::INT8); - node.output_buffer.allocate(); + if (kv_cache_resident()) { // GPU-resident shared cache (no snapshots) + void* p = cactus_metal_alloc_shared(node.output_buffer.byte_size); + if (p) node.output_buffer.set_external(p); else node.output_buffer.allocate(); + } else { + node.output_buffer.allocate(); + } std::memset(node.output_buffer.get_data(), 0, node.output_buffer.byte_size); auto* meta0 = get_meta(node.output_buffer, 0); diff --git a/cactus-graph/src/ops_math.cpp b/cactus-graph/src/ops_math.cpp index e87fc110b..8ac6e3d0a 100644 --- a/cactus-graph/src/ops_math.cpp +++ b/cactus-graph/src/ops_math.cpp @@ -1,5 +1,6 @@ #include "../cactus_graph.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include #include #include @@ -594,14 +595,24 @@ void compute_reshape_node(GraphNode& node, const std::vector(input_buffer.get_data())); + } else { + std::memcpy(node.output_buffer.get_data(), input_buffer.get_data(), input_buffer.byte_size); + } } void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { const auto& input_buf = get_input(node, 0, nodes, node_index_map); if (input_buf.precision == node.output_buffer.precision) { - std::memcpy(node.output_buffer.get_data(), input_buf.get_data(), input_buf.byte_size); + if (cactus_metal_active_mode()) // GPU: same-precision cast is a no-op view -> alias + node.output_buffer.set_external(const_cast(input_buf.get_data())); + else + std::memcpy(node.output_buffer.get_data(), input_buf.get_data(), input_buf.byte_size); return; } diff --git a/cactus-graph/src/ops_nn.cpp b/cactus-graph/src/ops_nn.cpp index bee9810ba..50d7c6029 100644 --- a/cactus-graph/src/ops_nn.cpp +++ b/cactus-graph/src/ops_nn.cpp @@ -1,5 +1,6 @@ #include "../cactus_graph.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include #include #include @@ -54,10 +55,17 @@ void compute_matmul_node(GraphNode& node, const std::vector(); CactusQuantMatrix mat = rhs_buffer.to_cq_matrix(); - if (rhs_buffer.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) + const bool is_orthogonal = (rhs_buffer.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) != 0; + static const bool gpu_matmul_env = + (std::getenv("CACTUS_GPU_MATMUL") != nullptr) && cactus_metal_available(); + if ((backend == ComputeBackend::GPU || gpu_matmul_env) && !is_orthogonal) { + // GPU fast path (M==1 4-bit Hadamard); falls back to CPU internally otherwise. + cactus_metal_quant_matmul(&mat, lhs, static_cast(M), output); + } else if (is_orthogonal) { cactus_quant_orthogonal_matmul(&mat, lhs, static_cast(M), output); - else + } else { cactus_quant_matmul(&mat, lhs, static_cast(M), output); + } } else { if (lhs_buffer.precision != Precision::FP16) { throw std::runtime_error("FP16 matmul requires FP16 activations (got precision " + std::to_string(static_cast(lhs_buffer.precision)) + ")"); diff --git a/cactus-graph/src/ops_tensor.cpp b/cactus-graph/src/ops_tensor.cpp index 857c677b9..c33a59996 100644 --- a/cactus-graph/src/ops_tensor.cpp +++ b/cactus-graph/src/ops_tensor.cpp @@ -76,15 +76,15 @@ void compute_slice_node(GraphNode& node, const std::vector(input_buffer.get_data()); if (!base_ptr) { throw std::runtime_error("Slice input buffer is not available"); } - const size_t byte_offset = PrecisionTraits::byte_offset_of(input_buffer.precision, slice_start * dims0.inner); + const size_t byte_offset = PrecisionTraits::byte_offset_of(input_buffer.precision, slice_start * dims.inner); node.output_buffer.set_external(base_ptr + byte_offset); node.output_buffer.precision = input_buffer.precision; @@ -101,8 +101,6 @@ void compute_slice_node(GraphNode& node, const std::vector static_cast(ComputeBackend::NPU)) { + if (backend_val > static_cast(ComputeBackend::GPU)) { throw std::runtime_error("Graph file corrupted: invalid backend"); } params.backend = static_cast(backend_val); diff --git a/cactus-kernels/CMakeLists.txt b/cactus-kernels/CMakeLists.txt index 5e1a805d2..0e57afeaa 100644 --- a/cactus-kernels/CMakeLists.txt +++ b/cactus-kernels/CMakeLists.txt @@ -65,6 +65,23 @@ if(APPLE) set(CMAKE_OSX_ARCHITECTURES "arm64") target_compile_definitions(cactus_kernels PUBLIC ACCELERATE_NEW_LAPACK) target_link_libraries(cactus_kernels PUBLIC "-framework Accelerate") + enable_language(OBJCXX) + set(_msl_in ${CMAKE_CURRENT_SOURCE_DIR}/src/cactus_kernels.metal) + set(_msl_out ${CMAKE_CURRENT_BINARY_DIR}/generated/cactus_kernels_msl.h) + add_custom_command( + OUTPUT ${_msl_out} + COMMAND ${CMAKE_COMMAND} -DIN=${_msl_in} -DOUT=${_msl_out} -DDELIM=CACTUS_MSL + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/embed_msl.cmake + DEPENDS ${_msl_in} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/embed_msl.cmake + VERBATIM) + add_custom_target(cactus_msl_header DEPENDS ${_msl_out}) + target_sources(cactus_kernels PRIVATE src/metal_backend.mm) + add_dependencies(cactus_kernels cactus_msl_header) + target_include_directories(cactus_kernels PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/generated) + set_source_files_properties(src/metal_backend.mm PROPERTIES COMPILE_FLAGS "-fobjc-arc") + target_link_libraries(cactus_kernels PUBLIC "-framework Metal" "-framework Foundation") +else() + target_sources(cactus_kernels PRIVATE src/metal_backend_stub.cpp) endif() if((BUILD_TESTING OR CACTUS_BUILD_TESTS) AND CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) diff --git a/cactus-kernels/cmake/embed_msl.cmake b/cactus-kernels/cmake/embed_msl.cmake new file mode 100644 index 000000000..12cab3bb1 --- /dev/null +++ b/cactus-kernels/cmake/embed_msl.cmake @@ -0,0 +1,8 @@ +file(READ "${IN}" SRC) +if(SRC MATCHES "\\)${DELIM}\"") + message(FATAL_ERROR "embed_msl: source contains the raw-string delimiter )${DELIM}\" - choose another DELIM") +endif() +file(WRITE "${OUT}" + "static const char* kCactusMSL = R\"${DELIM}(\n" + "${SRC}\n" + ")${DELIM}\";\n") diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h new file mode 100644 index 000000000..e622fed6f --- /dev/null +++ b/cactus-kernels/metal_backend.h @@ -0,0 +1,104 @@ +#ifndef CACTUS_METAL_BACKEND_H +#define CACTUS_METAL_BACKEND_H + +#if defined(__APPLE__) +#define CACTUS_HAS_METAL 1 +#else +#define CACTUS_HAS_METAL 0 +#endif + +#include "cactus_kernels.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif +bool cactus_metal_available(); +#ifdef __cplusplus +} +#endif + +void cactus_metal_set_active(bool active); +bool cactus_metal_active_mode(); + +bool cactus_metal_concurrent(); +void cactus_metal_barrier(); + +void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, + uint32_t M, __fp16* C); + +void cactus_metal_session_begin(); +void cactus_metal_session_sync(); +void cactus_metal_session_end(); + +void* cactus_metal_alloc_shared(size_t bytes); +void cactus_metal_free_shared(void* contents); + +bool cactus_metal_encode_copy(void* out, const void* in, size_t bytes); +bool cactus_metal_encode_binary(int op_type, void* out, const void* a, const void* b, size_t n); +bool cactus_metal_encode_scalar(int op_type, void* out, const void* in, size_t n, float param); +bool cactus_metal_encode_unary(int op_type, void* out, const void* in, size_t n); +bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, size_t n, float scale); +bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const void* sin, + uint32_t heads, uint32_t head_dim); +bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, + size_t rows, size_t dim, float eps); +bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* weight, const void* res, + size_t rows, size_t dim, float eps); +bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3); +bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_prec, size_t n); +bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W); +bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M); +bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W); +bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* code, + const CactusQuantMatrix* W); +bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W); +bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const CactusQuantMatrix* W); +bool cactus_metal_encode_embedding_ortho_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M); +bool cactus_metal_encode_embedding_hadamard_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M); +bool cactus_metal_encode_gather_f16(void* out, const void* table, size_t table_bytes, const uint32_t* rows, uint32_t M, uint32_t D); + +bool cactus_metal_encode_attention_i8( + void* out, const void* q, const void* knew, const void* vnew, + const void* kc, const void* vc, const void* ks, const void* vs, + uint32_t num_q_heads, uint32_t num_kv_heads, uint32_t head_dim, uint32_t v_hdim, + uint32_t history_len, uint32_t total_keys, uint32_t kv_start, uint32_t kv_end, + float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes); + +bool cactus_metal_encode_attention_i8_prefill( + void* out, const void* q, const void* knew, const void* vnew, + const void* kc, const void* vc, const void* ks, const void* vs, + uint32_t num_q_heads, uint32_t num_kv_heads, uint32_t head_dim, uint32_t v_hdim, + uint32_t history_len, uint32_t new_len, uint32_t q_pos0, uint32_t window, uint32_t is_causal, uint32_t M, + float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes, + uint32_t sink, uint32_t ring); + +bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* oshape, + const uint32_t* sstride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes); + +bool cactus_metal_encode_bcast_binary(int op, void* out, const void* a, const void* b, + const uint32_t* oshape, const uint32_t* astride, const uint32_t* bstride, uint32_t ndim, uint32_t total, + size_t a_bytes, size_t b_bytes, size_t out_bytes); + +bool cactus_metal_encode_strided_scatter(void* out, const void* in, const uint32_t* ishape, + const uint32_t* ostride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes); + +bool cactus_metal_encode_kv_append_i8(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, + size_t src_bytes, size_t int8_bytes, size_t scale_bytes); +bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, + uint32_t group_size, size_t src_bytes, size_t int8_bytes, size_t scale_bytes); +bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, + uint32_t group_size, uint32_t M, size_t src_bytes, size_t int8_bytes, size_t scale_bytes); + +bool cactus_metal_encode_kv_append_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, + size_t src_bytes, size_t int8_bytes, size_t scale_bytes); + +bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, + uint32_t sink, uint32_t W, size_t src_bytes, size_t int8_bytes, size_t scale_bytes); + +#endif diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal new file mode 100644 index 000000000..73f97cc4c --- /dev/null +++ b/cactus-kernels/src/cactus_kernels.metal @@ -0,0 +1,1191 @@ +#include +using namespace metal; +#define ROWS 8u + +kernel void cq4_transform( + device const half* x [[buffer(0)]], + device const half* recip [[buffer(1)]], + device const char* lsign [[buffer(2)]], + device const char* rsign [[buffer(3)]], + device const uint* perm [[buffer(4)]], + device half* code [[buffer(5)]], + constant uint& gs [[buffer(6)]], + uint g [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* z [[threadgroup(0)]]) +{ + + for (uint k=t; k=N) return; + uint K = num_groups*gs; + float acc=0; + for (uint base=lane*CQ4_VPL; base>4)&0xF] + + (float)c.z*cb[(ww>>8)&0xF] + (float)c.w*cb[(ww>>12)&0xF]; + } + acc += (float)norms[(size_t)n*num_groups+g]*p; + } + acc=simd_sum(acc); + if (lane==0) y[n]=(half)acc; +} + +kernel void cq4_transform_m( + device const half* x [[buffer(0)]], + device const half* recip [[buffer(1)]], + device const char* lsign [[buffer(2)]], + device const char* rsign [[buffer(3)]], + device const uint* perm [[buffer(4)]], + device half* code [[buffer(5)]], + constant uint& gs [[buffer(6)]], + constant uint& K [[buffer(7)]], + uint pos [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* z [[threadgroup(0)]]) +{ + uint ng = K/gs; + uint g = pos % ng, row = pos / ng; + size_t xb = (size_t)row*K + (size_t)g*gs; + for (uint k=t; k=N) return; + uint m0 = tg.y*CQ4_MT; + uint K = num_groups*gs; + float acc[CQ4_MT]; + for (uint mi=0; mi>4)&0xF]; + wv[q*4+2]=cb[(ww>>8)&0xF]; wv[q*4+3]=cb[(ww>>12)&0xF]; } + for (uint mi=0; mi C00=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C01=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C02=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C03=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C10=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C11=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C12=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C13=make_filled_simdgroup_matrix(0.f); + threadgroup half* Ac=As0; threadgroup half* Bc=Bs0; + threadgroup half* An=As1; threadgroup half* Bn=Bs1; + { + uint g = 0u, e0 = 0u; + for (uint i=tl; i<64u*32u; i+=128u){ + uint r=i>>5, k=i&31u; + Ac[r*ldA+k] = (m0+r>5, col=i&31u, n=n0+col; + float v=0.f; + if (n>1)]; + uint nib=(e&1u)?(uint)(by>>4):(uint)(by&0xF); + v=(float)norms[(size_t)n*num_groups+g]*cb[nib]; + } + Bc[k*ldB+col]=(half)v; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint k0=0; k0>5, k=i&31u; + An[r*ldA+k] = (m0+r>5, col=i&31u, n=n0+col; + float v=0.f; + if (n>1)]; + uint nib=(e&1u)?(uint)(by>>4):(uint)(by&0xF); + v=(float)norms[(size_t)n*num_groups+g]*cb[nib]; + } + Bn[k*ldB+col]=(half)v; + } + } + for (uint kk=0; kk<32u; kk+=8u){ + simdgroup_matrix A0,A1,B0,B1,B2,B3; + simdgroup_load(A0,&Ac[(rb)*ldA+kk],ldA); + simdgroup_load(A1,&Ac[(rb+8u)*ldA+kk],ldA); + simdgroup_load(B0,&Bc[kk*ldB+0u],ldB); + simdgroup_load(B1,&Bc[kk*ldB+8u],ldB); + simdgroup_load(B2,&Bc[kk*ldB+16u],ldB); + simdgroup_load(B3,&Bc[kk*ldB+24u],ldB); + simdgroup_multiply_accumulate(C00,A0,B0,C00); + simdgroup_multiply_accumulate(C01,A0,B1,C01); + simdgroup_multiply_accumulate(C02,A0,B2,C02); + simdgroup_multiply_accumulate(C03,A0,B3,C03); + simdgroup_multiply_accumulate(C10,A1,B0,C10); + simdgroup_multiply_accumulate(C11,A1,B1,C11); + simdgroup_multiply_accumulate(C12,A1,B2,C12); + simdgroup_multiply_accumulate(C13,A1,B3,C13); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + threadgroup half* t; + t=Ac; Ac=An; An=t; + t=Bc; Bc=Bn; Bn=t; + } + simdgroup_store(C00,&Cs[(rb)*32u + 0u],32u); + simdgroup_store(C01,&Cs[(rb)*32u + 8u],32u); + simdgroup_store(C02,&Cs[(rb)*32u + 16u],32u); + simdgroup_store(C03,&Cs[(rb)*32u + 24u],32u); + simdgroup_store(C10,&Cs[(rb+8u)*32u + 0u],32u); + simdgroup_store(C11,&Cs[(rb+8u)*32u + 8u],32u); + simdgroup_store(C12,&Cs[(rb+8u)*32u + 16u],32u); + simdgroup_store(C13,&Cs[(rb+8u)*32u + 24u],32u); + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint i=tl; i<64u*32u; i+=128u){ + uint r=i>>5, c=i&31u; + if (m0+r=K) return; + float acc = 0; + for (uint k=0; k>1]; uint idx=(i&1u)?(uint)(b>>4):(uint)(b&0xF); dq[i]=(float)codebook[idx]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + float nrm = (float)norms[row]; + for (uint j=t; j>1]; uint idx=(k&1u)?(uint)(b>>4):(uint)(b&0xF); + uint dst=perm[k]; z[dst]=(float)codebook[idx]*(float)rsign[dst]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint h=1; h>1)]; uint idx=(ki&1u)?(uint)(b>>4):(uint)(b&0xF); v=codebook[idx]; } + As[e]=v; + } + for (uint e=tid; e>1]; uint idx=(k&1u)?(uint)(b>>4):(uint)(b&0xF); uint dst=perm[k]; z[dst]=(float)codebook[idx]*(float)rsign[dst]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint h=1;h=n) return; + uint row=i/D, d=i%D; + out[i] = table[(size_t)rows[row]*D + d]; +} + +kernel void copy_bytes(device const uchar* in [[buffer(0)]], device uchar* out [[buffer(1)]], + constant uint& n [[buffer(2)]], uint i [[thread_position_in_grid]]) { + if (i=n) return; float av=(float)a[i], bv=(float)b[i], r; + switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; default: r=av+bv; } + if (op==1) r=clamp(r,-65500.0f,65500.0f); + y[i]=(half)r; +} + +kernel void rope_f16(device const half* x [[buffer(0)]], device half* out [[buffer(1)]], + device const half* cs [[buffer(2)]], device const half* sn [[buffer(3)]], + constant uint& heads [[buffer(4)]], constant uint& hd [[buffer(5)]], + uint gid [[thread_position_in_grid]]) { + uint total=heads*hd; if (gid>=total) return; + uint d=gid%hd, h=gid/hd, hh=hd/2; + float rot = (d=n) return; float v=(float)in[i], r; + switch(op){ case 0: r=v+p; break; case 1: r=v-p; break; case 3: r=v/p; break; default: r=v*p; } + y[i]=(half)r; +} + +kernel void unary_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], + uint i [[thread_position_in_grid]]) { + if (i>=n) return; float x=(float)in[i], r; + if (op==0){ float c=0.7978845608028654f*(x+0.044715f*x*x*x); r=0.5f*x*(1.0f+precise::tanh(c)); } + else if (op==1) r=precise::tanh(x); + else if (op==2) r=x/(1.0f+precise::exp(-x)); + else r=max(x,0.0f); + y[i]=(half)r; +} + +kernel void swiglu_f16(device const half* gate [[buffer(0)]], device const half* up [[buffer(1)]], + device half* y [[buffer(2)]], constant uint& n [[buffer(3)]], + constant float& scale [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i>=n) return; + float x=(float)gate[i]; + float c=0.7978845608028654f*(x+0.044715f*x*x*x); + half g1=(half)(0.5f*x*(1.0f+precise::tanh(c))); + half g2=(half)((float)g1*scale); + y[i]=(half)((float)g2*(float)up[i]); +} + +kernel void rms_norm_f16(device const half* in [[buffer(0)]], device const half* w [[buffer(1)]], + device half* y [[buffer(2)]], constant uint& dim [[buffer(3)]], + constant float& eps [[buffer(4)]], + uint row [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], + uint nt [[threads_per_threadgroup]], threadgroup float* red [[threadgroup(0)]]) { + device const half* x = in + (size_t)row*dim; + device half* o = y + (size_t)row*dim; + float partial=0; for (uint i=t;i0; s>>=1){ if (t0; s>>=1){ if (t=total) return; + uint rem=i, src=base; + for (int d=int(ndim)-1; d>=0; --d){ uint c=rem%oshape[d]; rem/=oshape[d]; src+=c*sstride[d]; } + out[i]=in[src]; +} + +kernel void kv_append_i8(device const half* src [[buffer(0)]], device char* int8base [[buffer(1)]], + device float* scalebase [[buffer(2)]], constant uint& kv_heads [[buffer(3)]], + constant uint& hdim [[buffer(4)]], constant uint& current_len [[buffer(5)]], constant uint& group_size [[buffer(6)]], + uint gid [[thread_position_in_grid]]) { + uint num_groups = (hdim + group_size - 1)/group_size; + if (gid >= kv_heads*num_groups) return; + uint h = gid / num_groups, g = gid % num_groups; + uint gstart = g*group_size, gcount = min(group_size, hdim - gstart); + device const half* hs = src + (size_t)h*hdim + gstart; + float maxabs = 0; + for (uint k=0;k= remaining*per) return; + uint s=gid/per, hg=gid%per, h=hg/num_groups, g=hg%num_groups; + uint gstart=g*group_size, gcount=min(group_size, hdim-gstart); + uint i8s=kv_heads*hdim, scs=kv_heads*num_groups, srcseq=shift_src+s; + device const char* sp=int8base+(size_t)srcseq*i8s+(size_t)h*hdim+gstart; + device char* dp=scr_i8+(size_t)s*i8s+(size_t)h*hdim+gstart; + for (uint k=0;k= (remaining+1)*per) return; + uint s=gid/per, hg=gid%per, h=hg/num_groups, g=hg%num_groups; + uint gstart=g*group_size, gcount=min(group_size, hdim-gstart); + uint i8s=kv_heads*hdim, scs=kv_heads*num_groups, dstseq=keep_sink+s; + device char* dp=int8base+(size_t)dstseq*i8s+(size_t)h*hdim+gstart; + if (s < remaining) { + device const char* sp=scr_i8+(size_t)s*i8s+(size_t)h*hdim+gstart; + for (uint k=0;k= (remaining+M)*per) return; + uint s=gid/per, hg=gid%per, h=hg/num_groups, g=hg%num_groups; + uint gstart=g*group_size, gcount=min(group_size, hdim-gstart); + uint i8s=kv_heads*hdim, scs=kv_heads*num_groups, dstseq=keep_sink+s; + device char* dp=int8base+(size_t)dstseq*i8s+(size_t)h*hdim+gstart; + if (s < remaining) { + device const char* sp=scr_i8+(size_t)s*i8s+(size_t)h*hdim+gstart; + for (uint k=0;k=total) return; + uint rem=i, dst=base; + for (int d=int(ndim)-1; d>=0; --d){ uint c=rem%ishape[d]; rem/=ishape[d]; dst+=c*ostride[d]; } + out[dst]=in[i]; +} + +kernel void bcast_binary_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], + device half* out [[buffer(2)]], constant uint* oshape [[buffer(3)]], + constant uint* astride [[buffer(4)]], constant uint* bstride [[buffer(5)]], + constant uint& ndim [[buffer(6)]], constant uint& total [[buffer(7)]], constant int& op [[buffer(8)]], + uint i [[thread_position_in_grid]]) { + if (i>=total) return; + uint rem=i, ai=0, bi=0; + for (int d=int(ndim)-1; d>=0; --d){ uint c=rem%oshape[d]; rem/=oshape[d]; ai+=c*astride[d]; bi+=c*bstride[d]; } + float av=(float)a[ai], bv=(float)b[bi], r; + switch(op){ case 2:r=av-bv;break; case 3:r=av*bv;break; case 4:r=av/bv;break; default:r=av+bv; } + if (op==1) r=clamp(r,-65500.0f,65500.0f); + out[i]=(half)r; +} + +kernel void attn_decode_i8( + device const half* q [[buffer(0)]], + device const half* knew [[buffer(1)]], + device const half* vnew [[buffer(2)]], + device const char* kc [[buffer(3)]], + device const char* vc [[buffer(4)]], + device const float* ks [[buffer(5)]], + device const float* vs [[buffer(6)]], + device half* out [[buffer(7)]], + constant uint& num_q_heads [[buffer(8)]], constant uint& num_kv_heads [[buffer(9)]], + constant uint& head_dim [[buffer(10)]], constant uint& v_hdim [[buffer(11)]], + constant uint& history_len [[buffer(12)]], constant float& scale [[buffer(13)]], + constant uint& kv_start [[buffer(14)]], constant uint& kv_end [[buffer(15)]], + uint h [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], threadgroup float* sc [[threadgroup(0)]]) +{ + uint kvh = h / (num_q_heads / num_kv_heads); + uint ngK = (head_dim + 31u)/32u, ngV = (v_hdim + 31u)/32u; + device const half* qh = q + (size_t)h*head_dim; + threadgroup float* red = sc + kv_end; + + float lmax = -INFINITY; + for (uint k = kv_start + t; k < kv_end; k += T) { + float dot = 0; + if (k < history_len) { + device const char* kk = kc + ((size_t)k*num_kv_heads + kvh)*head_dim; + device const float* kss = ks + ((size_t)k*num_kv_heads + kvh)*ngK; + for (uint d=0; d0;s>>=1){ if(t0;s>>=1){ if(t 0 ? 1.0f/red[0] : 0.0f; + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint d = t; d < v_hdim; d += T) { + float acc=0; + for (uint k=kv_start; k= cend) { + if (t == 0) { part_m[(size_t)h*C+c] = -INFINITY; part_l[(size_t)h*C+c] = 0.0f; } + for (uint d=t; d 0 ? 1.0f/gls[0] : 0.0f; + for (uint d=t; d 0u) ? sinkN : 0u; + uint Wsr = S + ringR; + uint rstart, nactive; + if (ringR > 0u && kv_end > Wsr) { rstart = kv_end - ringR; nactive = Wsr; } + else { rstart = S; nactive = kv_end; } + device const half* qh = q + ((size_t)m*num_q_heads + h)*head_dim; + threadgroup float* red = sc + maxsc; + if (nactive == 0u) { + for (uint d=t; d 0u) { + float lmax = -INFINITY; + for (uint j = t; j < nactive; j += T) { + uint k = (j < S) ? j : rstart + (j - S); + float dot = 0; + if (k < history_len) { + uint slot = (ringR > 0u && k >= Wsr) ? (S + ((k - S) % ringR)) : k; + device const char* kk = kc + ((size_t)slot*num_kv_heads + kvh)*head_dim; + device const float* kss = ks + ((size_t)slot*num_kv_heads + kvh)*ngK; + for (uint d=0; d0;s>>=1){ if(t0;s>>=1){ if(t 0 ? 1.0f/red[0] : 0.0f; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint d = t; d < v_hdim; d += T) { + float acc=0; + for (uint j=0; j 0u && k >= Wsr) ? (S + ((k - S) % ringR)) : k; + device const char* vvv = vc + ((size_t)slot*num_kv_heads+kvh)*v_hdim; + device const float* vss = vs + ((size_t)slot*num_kv_heads+kvh)*ngV; + vv = (float)vvv[d]*vss[d/32]; + } else { + device const half* vvv = vnew + ((size_t)(k-history_len)*num_kv_heads+kvh)*v_hdim; + vv = (float)vvv[d]; + } + acc += sc[j]*vv; + } + out[((size_t)m*num_q_heads + h)*v_hdim + d] = (half)(acc*inv); + } + } else { + uint TILE = maxsc; + float rmax = -INFINITY, rsum = 0.0f; + float acc[8]; + for (uint a=0;a<8;++a) acc[a]=0.0f; + for (uint tile=0; tile0;s>>=1){ if(t0;s>>=1){ if(t 0 ? 1.0f/rsum : 0.0f; + uint ai=0; + for (uint d=t; d> 5, lane = tl & 31u; + uint h = sg >> 1, hf = sg & 1u, kvh = 0u; + uint vd0 = hf*256u; + uint m0 = tgx*QB; + uint total_keys = history_len + new_len; + + uint pos_last = q_pos0 + m0 + (QB-1u); + uint kv_end_tile = min(total_keys, pos_last + 1u); + simdgroup_matrix O[NFH]; + + for (uint f=0; f(0.f); + if (hf==0u && lane < QB) { mrun[h*QB+lane] = -INFINITY; lrun[h*QB+lane]=0.0f; } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint k0=0; k0 C0=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C1=C0,C2=C0,C3=C0,C4=C0,C5=C0,C6=C0,C7=C0; + for (uint c=0; c>5, d=i&31u; + Qs[h*QB*LD + r*LD + d] = (m0+r A,B0,B1,B2,B3,B4,B5,B6,B7; + simdgroup_load(A, &Qs[h*QB*LD + kk], LD); + simdgroup_load(B0,&Ks[kk*LDK + 0u], LDK); + simdgroup_load(B1,&Ks[kk*LDK + 8u], LDK); + simdgroup_load(B2,&Ks[kk*LDK + 16u],LDK); + simdgroup_load(B3,&Ks[kk*LDK + 24u],LDK); + simdgroup_load(B4,&Ks[kk*LDK + 32u],LDK); + simdgroup_load(B5,&Ks[kk*LDK + 40u],LDK); + simdgroup_load(B6,&Ks[kk*LDK + 48u],LDK); + simdgroup_load(B7,&Ks[kk*LDK + 56u],LDK); + simdgroup_multiply_accumulate(C0,A,B0,C0); + simdgroup_multiply_accumulate(C1,A,B1,C1); + simdgroup_multiply_accumulate(C2,A,B2,C2); + simdgroup_multiply_accumulate(C3,A,B3,C3); + simdgroup_multiply_accumulate(C4,A,B4,C4); + simdgroup_multiply_accumulate(C5,A,B5,C5); + simdgroup_multiply_accumulate(C6,A,B6,C6); + simdgroup_multiply_accumulate(C7,A,B7,C7); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (hf==0u) { + simdgroup_store(C0,&Ss[h*QB*BK + 0u], BK); + simdgroup_store(C1,&Ss[h*QB*BK + 8u], BK); + simdgroup_store(C2,&Ss[h*QB*BK + 16u],BK); + simdgroup_store(C3,&Ss[h*QB*BK + 24u],BK); + simdgroup_store(C4,&Ss[h*QB*BK + 32u],BK); + simdgroup_store(C5,&Ss[h*QB*BK + 40u],BK); + simdgroup_store(C6,&Ss[h*QB*BK + 48u],BK); + simdgroup_store(C7,&Ss[h*QB*BK + 56u],BK); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (hf==0u && lane < QB) { + uint r=lane, qpos=q_pos0+m0+r; + threadgroup float* srow=&Ss[h*QB*BK + r*BK]; + float tmax=-INFINITY; + for (uint k=0;k-INFINITY)?exp(srow[k]-mnew):0.0f; + srow[k]=e; tsum+=e; + } + lrun[h*QB+r]=lrun[h*QB+r]*resc+tsum; + mrun[h*QB+r]=mnew; resc8[h*QB+r]=resc; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (lane == 0u) { + int nr = 0; + for (uint r=0; r>3; Rsc[sg*QB*8u + e] *= resc8[h*QB+r]; } + simdgroup_barrier(mem_flags::mem_threadgroup); + simdgroup_load(O[f], &Rsc[sg*QB*8u], 8u); + simdgroup_barrier(mem_flags::mem_threadgroup); + } + } + + if (hf==0u) for (uint i=lane; i>5, d=i&31u; + half val=(half)0; + if (k P,V0; + simdgroup_load(P, &Ps[h*QB*LDP + kk], LDP); + simdgroup_load(V0, &Vs[hf*BK*LD + kk*LD + fc], LD); + simdgroup_multiply_accumulate(O[f], P, V0, O[f]); + } + } + } + } + for (uint f=0; f>3, d=e&7u; + if (m0+r0?1.0f/lrun[h*QB+r]:0.0f; + out[((size_t)(m0+r)*num_q_heads + h)*VD + (vd0 + f*8u + d)] = (half)(Rsc[sg*QB*8u + e]*inv); + } + } + simdgroup_barrier(mem_flags::mem_threadgroup); + } +} + +kernel void kv_append_i8_m(device const half* src [[buffer(0)]], device char* int8base [[buffer(1)]], + device float* scalebase [[buffer(2)]], constant uint& kv_heads [[buffer(3)]], + constant uint& hdim [[buffer(4)]], constant uint& current_len [[buffer(5)]], + constant uint& group_size [[buffer(6)]], constant uint& M [[buffer(7)]], + uint gid [[thread_position_in_grid]]) { + uint num_groups = (hdim + group_size - 1)/group_size, per = kv_heads*num_groups; + if (gid >= M*per) return; + uint i = gid / per, hg = gid % per, h = hg / num_groups, g = hg % num_groups; + uint gstart = g*group_size, gcount = min(group_size, hdim - gstart); + device const half* hs = src + (size_t)i*kv_heads*hdim + (size_t)h*hdim + gstart; + float maxabs = 0; + for (uint k=0;k= M*per) return; + uint i = gid / per, hg = gid % per, h = hg / num_groups, g = hg % num_groups; + uint gstart = g*group_size, gcount = min(group_size, hdim - gstart); + device const half* hs = src + (size_t)i*kv_heads*hdim + (size_t)h*hdim + gstart; + float maxabs = 0; + for (uint k=0;k sink) ? (W - sink) : 1u; + uint slot = (pos < W) ? pos : sink + ((pos - sink) % R); + device char* dst = int8base + (size_t)slot*i8s + (size_t)h*hdim + gstart; + for (uint k=0;k b) { s = b; b = v; bi = i; } + else if (v > s) { s = v; } + } + sb[t] = b; si[t] = bi; ss[t] = s; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = T >> 1; stride > 0u; stride >>= 1) { + if (t < stride) { + float ab = sb[t], asx = ss[t]; uint ai = si[t]; + float bb = sb[t+stride], bsx = ss[t+stride]; uint bidx = si[t+stride]; + if (ab > bb || (ab == bb && ai <= bidx)) { sb[t] = ab; si[t] = ai; ss[t] = max(asx, bb); } + else { sb[t] = bb; si[t] = bidx; ss[t] = max(bsx, ab); } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (t == 0u) { out3[0] = sb[0]; out3[1] = ss[0]; out3[2] = (float)si[0]; } +} diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm new file mode 100644 index 000000000..7d67aef47 --- /dev/null +++ b/cactus-kernels/src/metal_backend.mm @@ -0,0 +1,1017 @@ + + +#include "metal_backend.h" + +#if CACTUS_HAS_METAL + +#import +#import +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cactus_kernels_msl.h" + +namespace { + +struct MetalCtx { + id dev = nil; + id queue = nil; + id psoT = nil; + id psoTsimd = nil; + id psoG = nil; + id psoTm = nil, psoGm = nil; + id psoGmma = nil; + id psoRotate = nil; + id psoEmbO = nil, psoEmbH = nil; + id psoEmbOm = nil, psoEmbHm = nil; + id psoCopy=nil, psoBinary=nil, psoScalar=nil, psoUnary=nil, psoRms=nil, psoSwiglu=nil, psoRmsAdd=nil; + id psoCF16F32=nil, psoCF32F16=nil, psoCI8F16=nil, psoCF16I8=nil; + id psoAttn=nil, psoStrided=nil, psoScatter=nil, psoBcast=nil, psoKvAppend=nil; + id psoAttnP=nil, psoAttnC=nil; + id psoAttnPre=nil, psoAttnPreMma2=nil; + id psoKvAppendM=nil, psoKvAppendRingM=nil; + id psoSlideS=nil, psoSlideR=nil, psoSlideRM=nil; + id psoRope=nil; + id psoArgmax=nil; + id psoGather=nil; + id dummy=nil; + bool ok = false; + + MetalCtx() { @autoreleasepool { + dev = MTLCreateSystemDefaultDevice(); + if (!dev) return; + queue = [dev newCommandQueue]; + NSError* err = nil; + id lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kCactusMSL] + options:nil error:&err]; + + if (!lib) { if (err) fprintf(stderr,"[cactus-metal] MSL compile failed: %s\n",[[err localizedDescription] UTF8String]); return; } + auto pso = [&](const char* name) -> id { + NSError* e=nil; + id f=[lib newFunctionWithName:[NSString stringWithUTF8String:name]]; + id p = f ? [dev newComputePipelineStateWithFunction:f error:&e] : nil; + if (!p) fprintf(stderr,"[cactus-metal] pipeline '%s' failed: %s\n", name, e?[[e localizedDescription] UTF8String]:"function not found"); + return p; + }; + psoT=pso("cq4_transform"); psoTsimd=pso("cq4_transform_simd"); psoG=pso("cq4_gemv"); + psoTm=pso("cq4_transform_m"); psoGm=pso("cq4_gemm"); psoGmma=pso("cq4_gemm_mma"); + psoRotate=pso("lmhead_rotate"); + psoEmbO=pso("emb_ortho"); psoEmbH=pso("emb_hadamard"); + psoEmbOm=pso("emb_ortho_m"); psoEmbHm=pso("emb_hadamard_m"); + psoGather=pso("gather_f16"); + psoCopy=pso("copy_bytes"); psoBinary=pso("binary_f16"); psoScalar=pso("scalar_f16"); + psoUnary=pso("unary_f16"); psoRms=pso("rms_norm_f16"); psoSwiglu=pso("swiglu_f16"); psoRmsAdd=pso("rms_norm_add_f16"); + psoCF16F32=pso("cast_f16_f32"); psoCF32F16=pso("cast_f32_f16"); + psoCI8F16=pso("cast_i8_f16"); psoCF16I8=pso("cast_f16_i8"); + psoAttn=pso("attn_decode_i8"); psoStrided=pso("strided_copy_f16"); psoBcast=pso("bcast_binary_f16"); + psoAttnP=pso("attn_decode_i8_partial"); psoAttnC=pso("attn_decode_combine"); + psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoKvAppendM=pso("kv_append_i8_m"); + psoKvAppendRingM=pso("kv_append_ring_i8_m"); + psoSlideS=pso("kv_slide_save"); psoSlideR=pso("kv_slide_restore"); psoSlideRM=pso("kv_slide_restore_m"); + psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); + psoArgmax=pso("argmax_logits"); + dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; + ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 + &&psoAttn&&psoAttnP&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; + }} +}; + +MetalCtx& ctx() { static MetalCtx c; return c; } + +inline id buf(const void* p, size_t bytes) { + if (!p || bytes == 0) return nil; + return [ctx().dev newBufferWithBytes:p length:bytes options:MTLResourceStorageModeShared]; +} + +struct ResW { + id packed=nil, norms=nil, codebook=nil, lsign=nil, rsign=nil, perm=nil, recip=nil; + id rotation=nil; +}; +std::unordered_map g_resident; +std::mutex g_resident_mu; + +void deinterleave_4row(const CactusQuantMatrix* W, + uint8_t* packed_nat, __fp16* norms_nat) { + const uint32_t N=W->N, ng=W->num_groups, gs=W->group_size, pgb=(gs*4u+7u)/8u, NB=N/4; + std::memset(packed_nat, 0, (size_t)N*ng*pgb); + const uint8_t* il=W->packed_indices; + for (uint32_t nb=0; nbnorms[((size_t)nb*ng+g)*4 + r]; + uint8_t* row = packed_nat + ((size_t)n*ng+g)*pgb; + for (uint32_t v=0; v>1] |= (k0&1)?((lo&0xF)<<4):(lo&0xF); + row[k1>>1] |= (k1&1)?((lo>>4)<<4):(lo>>4); + row[k2>>1] |= (k2&1)?((hi&0xF)<<4):(hi&0xF); + row[k3>>1] |= (k3&1)?((hi>>4)<<4):(hi>>4); + } + } + } +} + +ResW& resident(const CactusQuantMatrix* W) { + std::lock_guard lk(g_resident_mu); + auto it = g_resident.find(W->packed_indices); + if (it != g_resident.end()) return it->second; + const uint32_t K=W->K, N=W->N, ng=W->num_groups, gs=W->group_size, pgb=(gs*4u+7u)/8u; + ResW r; + r.codebook = buf(W->codebook, 16*sizeof(__fp16)); + r.lsign = buf(W->left_signs, gs); + r.rsign = buf(W->right_signs, gs); + r.perm = buf(W->permutation, (size_t)gs*sizeof(uint32_t)); + r.recip = buf(W->input_scale_recip, (size_t)K*sizeof(__fp16)); + if (W->rotation) r.rotation = buf(W->rotation, (size_t)K*K*sizeof(__fp16)); + if (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) { + size_t pkb=(size_t)N*ng*pgb, nmb=(size_t)N*ng*sizeof(__fp16); + r.packed = [ctx().dev newBufferWithLength:pkb options:MTLResourceStorageModeShared]; + r.norms = [ctx().dev newBufferWithLength:nmb options:MTLResourceStorageModeShared]; + deinterleave_4row(W, (uint8_t*)r.packed.contents, (__fp16*)r.norms.contents); + } else { + r.packed = buf(W->packed_indices, (size_t)N*ng*pgb); + r.norms = buf(W->norms, (size_t)N*ng*sizeof(__fp16)); + } + return g_resident.emplace(W->packed_indices, r).first->second; +} + +id g_cmd = nil; +id g_enc = nil; +std::unordered_map>> g_free; +std::vector> g_pending; +std::map> g_shared; +bool g_active = false; +id g_code_buf = nil; +id g_code_buf_m = nil; + +inline size_t bucket(size_t b) { return (b + 4095) & ~size_t(4095); } + +static const bool g_concurrent = (std::getenv("CACTUS_GPU_CONCURRENT") != nullptr); + +id ensureEncoder() { + if (!g_enc) { + g_cmd = [ctx().queue commandBuffer]; + g_enc = g_concurrent ? [g_cmd computeCommandEncoderWithDispatchType:MTLDispatchTypeConcurrent] + : [g_cmd computeCommandEncoder]; + } + return g_enc; +} + +inline void barrier() { + static const bool force = (std::getenv("CACTUS_FORCE_BARRIER") != nullptr); + if (g_concurrent || force) [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; +} + +id recycled(size_t bytes) { + size_t bk = bucket(bytes ? bytes : 1); + auto it = g_free.find(bk); + id b; + if (it != g_free.end() && !it->second.empty()) { b = it->second.back(); it->second.pop_back(); } + else b = [ctx().dev newBufferWithLength:bk options:MTLResourceStorageModeShared]; + g_pending.push_back(b); + return b; +} + +std::pair, size_t> bufForPtrOff(const void* p, size_t bytes) { + uintptr_t a = reinterpret_cast(p); + auto it = g_shared.upper_bound(a); + if (it != g_shared.begin()) { + --it; + uintptr_t base = it->first; + if (a < base + static_cast(it->second.length)) + return { it->second, static_cast(a - base) }; + } + + static const bool g_replay = (std::getenv("CACTUS_GPU_REPLAY") != nullptr); + size_t nb = bytes ? bytes : 1; + if (g_replay) { + static std::unordered_map> g_snap; + auto sit = g_snap.find(p); + id b; + if (sit != g_snap.end() && (size_t)sit->second.length >= nb) b = sit->second; + else { b = [ctx().dev newBufferWithLength:nb options:MTLResourceStorageModeShared]; g_snap[p] = b; } + std::memcpy([b contents], p, nb); + return { b, 0 }; + } + id b = recycled(bytes); + std::memcpy([b contents], p, nb); + return { b, 0 }; +} + +inline void setBufAt(const void* p, size_t bytes, int idx) { + auto pr = bufForPtrOff(p, bytes); + [g_enc setBuffer:pr.first offset:pr.second atIndex:idx]; +} + +inline void setBufFresh(const void* p, size_t bytes, int idx) { + id b = recycled(bytes); + std::memcpy([b contents], p, bytes ? bytes : 1); + [g_enc setBuffer:b offset:0 atIndex:idx]; +} + +} + +bool cactus_metal_available() { return ctx().ok; } + +void cactus_metal_set_active(bool a) { g_active = a; } +bool cactus_metal_active_mode() { return ctx().ok && g_active; } + +bool cactus_metal_concurrent() { return g_concurrent; } +void cactus_metal_barrier() { if (g_concurrent && g_enc) [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; } + +void cactus_metal_session_begin() { } +static int g_sync_count = 0; +static double g_gpu_ms = 0; +void cactus_metal_session_sync() { + if (g_enc) { + [g_enc endEncoding]; [g_cmd commit]; [g_cmd waitUntilCompleted]; + g_gpu_ms += ([g_cmd GPUEndTime] - [g_cmd GPUStartTime]) * 1000.0; + g_enc = nil; g_cmd = nil; ++g_sync_count; + } + for (id b : g_pending) g_free[(size_t)b.length].push_back(b); + g_pending.clear(); +} +void cactus_metal_session_end() { + cactus_metal_session_sync(); + if (getenv("CACTUS_GPU_STATS")) { + fprintf(stderr, "[gpu-stats] syncs=%d on-GPU=%.3f ms\n", g_sync_count, g_gpu_ms); + g_sync_count = 0; g_gpu_ms = 0; + } +} + +void* cactus_metal_alloc_shared(size_t bytes) { + if (!ctx().ok) return nullptr; + size_t bk = bucket(bytes ? bytes : 1); + id b = nil; + auto it = g_free.find(bk); + if (it != g_free.end() && !it->second.empty()) { b = it->second.back(); it->second.pop_back(); } + else b = [ctx().dev newBufferWithLength:bk options:MTLResourceStorageModeShared]; + void* c = [b contents]; + g_shared[reinterpret_cast(c)] = b; + return c; +} +void cactus_metal_free_shared(void* contents) { + auto it = g_shared.find(reinterpret_cast(contents)); + if (it != g_shared.end()) { g_pending.push_back(it->second); g_shared.erase(it); } +} + +bool cactus_metal_encode_copy(void* out, const void* in, size_t bytes) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t n=(uint32_t)bytes; + [g_enc setComputePipelineState:ctx().psoCopy]; + setBufAt(in, bytes, 0); + setBufAt(out, bytes, 1); + [g_enc setBytes:&n length:4 atIndex:2]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_binary(int op, void* out, const void* a, const void* b, size_t n) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; + [g_enc setComputePipelineState:ctx().psoBinary]; + setBufAt(a, n*2, 0); setBufAt(b, n*2, 1); setBufAt(out, n*2, 2); + [g_enc setBytes:&nn length:4 atIndex:3]; [g_enc setBytes:&o length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_scalar(int op, void* out, const void* in, size_t n, float param) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; float p=param; + [g_enc setComputePipelineState:ctx().psoScalar]; + setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; [g_enc setBytes:&p length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_unary(int op, void* out, const void* in, size_t n) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; + [g_enc setComputePipelineState:ctx().psoUnary]; + setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; + [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, size_t n, float scale) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; float s=scale; + [g_enc setComputePipelineState:ctx().psoSwiglu]; + setBufAt(gate, n*2, 0); setBufAt(up, n*2, 1); setBufAt(out, n*2, 2); + [g_enc setBytes:&nn length:4 atIndex:3]; [g_enc setBytes:&s length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const void* sin, + uint32_t heads, uint32_t head_dim) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t total=heads*head_dim; + [g_enc setComputePipelineState:ctx().psoRope]; + setBufAt(x, (size_t)total*2, 0); setBufAt(out, (size_t)total*2, 1); + setBufAt(cos, (size_t)head_dim*2, 2); setBufAt(sin, (size_t)head_dim*2, 3); + [g_enc setBytes:&heads length:4 atIndex:4]; [g_enc setBytes:&head_dim length:4 atIndex:5]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, + size_t rows, size_t dim, float eps) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t d=(uint32_t)dim; float e=eps; + [g_enc setComputePipelineState:ctx().psoRms]; + setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(out, rows*dim*2, 2); + [g_enc setBytes:&d length:4 atIndex:3]; [g_enc setBytes:&e length:4 atIndex:4]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* weight, const void* res, + size_t rows, size_t dim, float eps) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t d=(uint32_t)dim; float e=eps; + [g_enc setComputePipelineState:ctx().psoRmsAdd]; + setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(res, rows*dim*2, 2); setBufAt(out, rows*dim*2, 3); + [g_enc setBytes:&d length:4 atIndex:4]; [g_enc setBytes:&e length:4 atIndex:5]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t V = vocab, T = 1024; + [g_enc setComputePipelineState:ctx().psoArgmax]; + setBufAt(logits, (size_t)vocab*2, 0); + setBufAt(out3, 3*sizeof(float), 1); + [g_enc setBytes:&V length:4 atIndex:2]; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:T*sizeof(uint) atIndex:1]; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:2]; + [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_prec, size_t n) { + if (!ctx().ok) return false; + + auto esz=[](int p){ return p==0?1: p==1?2: p==2?4: 0; }; + if (in_prec==out_prec) return cactus_metal_encode_copy(out, in, n*esz(in_prec)); + id pso=nil; + if (in_prec==1&&out_prec==2) pso=ctx().psoCF16F32; + else if (in_prec==2&&out_prec==1) pso=ctx().psoCF32F16; + else if (in_prec==0&&out_prec==1) pso=ctx().psoCI8F16; + else if (in_prec==1&&out_prec==0) pso=ctx().psoCF16I8; + else return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; + [g_enc setComputePipelineState:pso]; + setBufAt(in, n*esz(in_prec), 0); setBufAt(out, n*esz(out_prec), 1); + [g_enc setBytes:&nn length:4 atIndex:2]; + [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W) { + const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; + if (ctx().ok && (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->rotation && ng==1 && gs==K && (N%4)==0) { + static void* ortho_code = nullptr; static uint32_t ortho_code_k = 0; + if (ortho_code_k < K) { ortho_code = cactus_metal_alloc_shared((size_t)K*sizeof(__fp16)); ortho_code_k = K; } + if (ortho_code) return cactus_metal_encode_quant_matmul_ortho(out, lhs, ortho_code, W); + } + bool fast = ctx().ok && W->bits==4 && gs>=128 && (gs%128)==0 && (N%4)==0 && + !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->input_scale_recip && W->left_signs && + W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; + if (!fast) return false; + ResW& rw = resident(W); + + size_t code_bytes = (size_t)ng*gs*sizeof(__fp16); + if (!g_code_buf || (size_t)g_code_buf.length < code_bytes) + g_code_buf = [ctx().dev newBufferWithLength:code_bytes options:MTLResourceStorageModeShared]; + ensureEncoder(); + bool simdT = (gs==128u && ctx().psoTsimd); + [g_enc setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; + setBufAt(lhs, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; + [g_enc setBuffer:rw.lsign offset:0 atIndex:2]; [g_enc setBuffer:rw.rsign offset:0 atIndex:3]; + [g_enc setBuffer:rw.perm offset:0 atIndex:4]; [g_enc setBuffer:g_code_buf offset:0 atIndex:5]; + [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; + barrier(); + [g_enc setComputePipelineState:ctx().psoG]; + [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:0 atIndex:1]; + [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; + setBufAt(out, (size_t)N*2, 4); + [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; + [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; + uint32_t ROWS=8; + [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { + if (!ctx().ok) return false; + const uint32_t gs=W->group_size, N=W->N; + bool fast = W->bits==4 && gs>=128 && (gs%128)==0 && (N%4)==0 && + !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->input_scale_recip && W->left_signs && + W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; + if (!fast) return false; + resident(W); + return true; +} + +bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M) { + if (M == 1) return cactus_metal_encode_quant_matmul(out, lhs, W); + const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; + bool fast = ctx().ok && W->bits==4 && gs>=128 && (gs%128)==0 && (N%4)==0 && + !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->input_scale_recip && W->left_signs && + W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; + if (!fast) return false; + ResW& rw = resident(W); + size_t code_bytes = (size_t)M*ng*gs*sizeof(__fp16); + if (!g_code_buf_m || (size_t)g_code_buf_m.length < code_bytes) + g_code_buf_m = [ctx().dev newBufferWithLength:code_bytes options:MTLResourceStorageModeShared]; + ensureEncoder(); + + [g_enc setComputePipelineState:ctx().psoTm]; + setBufAt(lhs, (size_t)M*K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; + [g_enc setBuffer:rw.lsign offset:0 atIndex:2]; [g_enc setBuffer:rw.rsign offset:0 atIndex:3]; + [g_enc setBuffer:rw.perm offset:0 atIndex:4]; [g_enc setBuffer:g_code_buf_m offset:0 atIndex:5]; + [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setBytes:&K length:4 atIndex:7]; + [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; + barrier(); + + bool mma = (ctx().psoGmma != nil); + [g_enc setComputePipelineState:(mma?ctx().psoGmma:ctx().psoGm)]; + [g_enc setBuffer:g_code_buf_m offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:0 atIndex:1]; + [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; + setBufAt(out, (size_t)M*N*2, 4); + [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; + [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; [g_enc setBytes:&M length:4 atIndex:9]; + if (mma) { + [g_enc dispatchThreadgroups:MTLSizeMake((N+31)/32,(M+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + } else { + uint32_t ROWS=8, MT=16; + [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,(M+MT-1)/MT,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + } + barrier(); + return true; +} + +bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* code, + const CactusQuantMatrix* W) { + if (!ctx().ok || !W->rotation || W->bits != 4) return false; + const uint32_t K=W->K, N=W->N, ng=W->num_groups, gs=W->group_size, pgb=(gs*4u+7u)/8u; + if (ng != 1 || gs != K || (N % 4) != 0) return false; + ResW& rw = resident(W); + if (!rw.rotation || !rw.packed) return false; + ensureEncoder(); + + [g_enc setComputePipelineState:ctx().psoRotate]; + setBufAt(act, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; + [g_enc setBuffer:rw.rotation offset:0 atIndex:2]; setBufAt(code, (size_t)K*2, 3); + [g_enc setBytes:&K length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(K,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + + [g_enc setComputePipelineState:ctx().psoG]; + setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:0 atIndex:1]; + [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; + setBufAt(out, (size_t)N*2, 4); + [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; + [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; + uint32_t ROWS=8; + [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W) { + if (!ctx().ok || !W->rotation || W->bits != 4) return false; + const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size; + if (ng != 1 || gs != K) return false; + ResW& rw = resident(W); + if (!rw.packed || !rw.rotation || !rw.codebook || !rw.norms || !rw.recip) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoEmbO]; + [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; + [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.recip offset:0 atIndex:3]; + [g_enc setBuffer:rw.rotation offset:0 atIndex:4]; setBufAt(out, (size_t)K*2, 5); + [g_enc setBytes:&K length:4 atIndex:6]; [g_enc setBytes:&row length:4 atIndex:7]; + uint32_t T = K>512u ? 512u : K; + [g_enc setThreadgroupMemoryLength:(size_t)K*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + barrier(); + return true; +} + +std::unordered_map g_resident_emb; +ResW& resident_emb_meta(const CactusQuantMatrix* W) { + std::lock_guard lk(g_resident_mu); + auto it = g_resident_emb.find(W->packed_indices); + if (it != g_resident_emb.end()) return it->second; + const uint32_t K=W->K, gs=W->group_size; + ResW r; + r.codebook = buf(W->codebook, 16*sizeof(__fp16)); + r.lsign = buf(W->left_signs, gs); + r.rsign = buf(W->right_signs, gs); + r.perm = buf(W->permutation, (size_t)gs*sizeof(uint32_t)); + r.recip = buf(W->input_scale_recip, (size_t)K*sizeof(__fp16)); + return g_resident_emb.emplace(W->packed_indices, r).first->second; +} + +bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const CactusQuantMatrix* W) { + if (!ctx().ok || W->bits != 4 || (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) return false; + const uint32_t K=W->K, gs=W->group_size, ng=W->num_groups, pgb=(gs*4u+7u)/8u; + if (gs > 256 || (gs & (gs-1)) != 0 || !W->packed_indices || !W->norms || !W->codebook + || !W->left_signs || !W->right_signs || !W->permutation || !W->input_scale_recip) return false; + ResW& rm = resident_emb_meta(W); + if (!rm.codebook || !rm.recip || !rm.lsign || !rm.rsign || !rm.perm) return false; + ensureEncoder(); + size_t row_bytes = (size_t)ng*pgb; + id prow = recycled(row_bytes); + std::memcpy([prow contents], (const uint8_t*)W->packed_indices + (size_t)row*ng*pgb, row_bytes); + id nrow = recycled((size_t)ng*sizeof(__fp16)); + std::memcpy([nrow contents], (const __fp16*)W->norms + (size_t)row*ng, (size_t)ng*sizeof(__fp16)); + [g_enc setComputePipelineState:ctx().psoEmbH]; + [g_enc setBuffer:prow offset:0 atIndex:0]; [g_enc setBuffer:rm.codebook offset:0 atIndex:1]; + [g_enc setBuffer:nrow offset:0 atIndex:2]; [g_enc setBuffer:rm.recip offset:0 atIndex:3]; + [g_enc setBuffer:rm.lsign offset:0 atIndex:4]; [g_enc setBuffer:rm.rsign offset:0 atIndex:5]; + [g_enc setBuffer:rm.perm offset:0 atIndex:6]; setBufAt(out, (size_t)K*2, 7); + [g_enc setBytes:&gs length:4 atIndex:8]; + [g_enc setThreadgroupMemoryLength:(size_t)gs*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(gs,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_embedding_ortho_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M) { + if (!ctx().ok || !ctx().psoEmbOm || !W->rotation || W->bits != 4) return false; + const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size; + if (ng != 1 || gs != K) return false; + ResW& rw = resident(W); + if (!rw.packed || !rw.rotation || !rw.codebook || !rw.norms || !rw.recip) return false; + ensureEncoder(); + id rb = recycled((size_t)M*sizeof(uint32_t)); + std::memcpy([rb contents], rows, (size_t)M*sizeof(uint32_t)); + [g_enc setComputePipelineState:ctx().psoEmbOm]; + [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; + [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.recip offset:0 atIndex:3]; + [g_enc setBuffer:rw.rotation offset:0 atIndex:4]; [g_enc setBuffer:rb offset:0 atIndex:5]; + setBufAt(out, (size_t)M*K*2, 6); [g_enc setBytes:&K length:4 atIndex:7]; [g_enc setBytes:&M length:4 atIndex:8]; + [g_enc dispatchThreadgroups:MTLSizeMake(((K+15)/16)*((M+15)/16),1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_embedding_hadamard_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M) { + if (!ctx().ok || !ctx().psoEmbHm || W->bits != 4 || (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) return false; + const uint32_t K=W->K, gs=W->group_size, ng=W->num_groups, pgb=(gs*4u+7u)/8u; + if (gs > 256 || (gs & (gs-1)) != 0 || !W->packed_indices || !W->norms || !W->codebook + || !W->left_signs || !W->right_signs || !W->permutation || !W->input_scale_recip) return false; + ResW& rm = resident_emb_meta(W); + if (!rm.codebook || !rm.recip || !rm.lsign || !rm.rsign || !rm.perm) return false; + ensureEncoder(); + size_t rowb = (size_t)ng*pgb; + id pk = recycled((size_t)M*rowb); + id nm = recycled((size_t)M*ng*sizeof(__fp16)); + for (uint32_t m=0; mpacked_indices+(size_t)rows[m]*rowb, rowb); + std::memcpy((__fp16*)[nm contents]+(size_t)m*ng, (const __fp16*)W->norms+(size_t)rows[m]*ng, (size_t)ng*sizeof(__fp16)); + } + [g_enc setComputePipelineState:ctx().psoEmbHm]; + [g_enc setBuffer:pk offset:0 atIndex:0]; [g_enc setBuffer:rm.codebook offset:0 atIndex:1]; + [g_enc setBuffer:nm offset:0 atIndex:2]; [g_enc setBuffer:rm.recip offset:0 atIndex:3]; + [g_enc setBuffer:rm.lsign offset:0 atIndex:4]; [g_enc setBuffer:rm.rsign offset:0 atIndex:5]; + [g_enc setBuffer:rm.perm offset:0 atIndex:6]; setBufAt(out, (size_t)M*K*2, 7); + [g_enc setBytes:&gs length:4 atIndex:8]; [g_enc setBytes:&ng length:4 atIndex:9]; [g_enc setBytes:&K length:4 atIndex:10]; + [g_enc setThreadgroupMemoryLength:(size_t)gs*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs,1,1)]; + barrier(); + return true; +} + +static std::unordered_map> g_readonly; +static id registerReadonly(const void* p, size_t bytes) { + if (!p) return nil; + auto it = g_readonly.find(p); + if (it != g_readonly.end()) return it->second; + uintptr_t a = (uintptr_t)p, base = a & ~(uintptr_t)16383u; + size_t wraplen = (((a - base) + bytes + 16383u) & ~(size_t)16383u); + id b = [ctx().dev newBufferWithBytesNoCopy:(void*)base length:wraplen + options:MTLResourceStorageModeShared deallocator:nil]; + if (b) g_readonly[p] = b; + return b; +} + +bool cactus_metal_encode_gather_f16(void* out, const void* table, size_t table_bytes, + const uint32_t* rows, uint32_t M, uint32_t D) { + if (!ctx().ok || !ctx().psoGather || M == 0 || D == 0) return false; + id tb = registerReadonly(table, table_bytes); + if (!tb) return false; + size_t toff = (uintptr_t)table & (uintptr_t)16383u; + ensureEncoder(); + id rb = recycled((size_t)M*sizeof(uint32_t)); + std::memcpy([rb contents], rows, (size_t)M*sizeof(uint32_t)); + uint32_t n = M*D; + [g_enc setComputePipelineState:ctx().psoGather]; + [g_enc setBuffer:tb offset:toff atIndex:0]; [g_enc setBuffer:rb offset:0 atIndex:1]; + setBufAt(out, (size_t)n*2, 2); + [g_enc setBytes:&D length:4 atIndex:3]; [g_enc setBytes:&n length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_attention_i8( + void* out, const void* q, const void* knew, const void* vnew, + const void* kc, const void* vc, const void* ks, const void* vs, + uint32_t num_q_heads, uint32_t num_kv_heads, uint32_t head_dim, uint32_t v_hdim, + uint32_t history_len, uint32_t total_keys, uint32_t kv_start, uint32_t kv_end, + float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes) { + if (!ctx().ok) return false; + if (kv_end <= kv_start || (num_q_heads % num_kv_heads) != 0) return false; + ensureEncoder(); + auto setCache = [&](const void* p, size_t bytes, int idx) { + + if (p && bytes) setBufAt(p, bytes, idx); + else [g_enc setBuffer:ctx().dummy offset:0 atIndex:idx]; + }; + auto setInputs = [&]() { + setBufAt(q, (size_t)num_q_heads*head_dim*2, 0); + if (total_keys > history_len && knew && vnew) { + setBufAt(knew, (size_t)num_kv_heads*head_dim*2, 1); + setBufAt(vnew, (size_t)num_kv_heads*v_hdim*2, 2); + } else { + [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + } + setCache(kc, kc_bytes, 3); setCache(vc, vc_bytes, 4); + setCache(ks, ks_bytes, 5); setCache(vs, vs_bytes, 6); + }; + + static const bool g_nosplit = (std::getenv("CACTUS_ATTN_NOSPLIT") != nullptr); + static const uint32_t CHUNK_BASE = std::getenv("CACTUS_ATTN_CHUNK") ? (uint32_t)atoi(std::getenv("CACTUS_ATTN_CHUNK")) : 96; + static const uint32_t C_MAX = std::getenv("CACTUS_ATTN_CMAX") ? (uint32_t)atoi(std::getenv("CACTUS_ATTN_CMAX")) : 64; + const uint32_t R = kv_end - kv_start; + uint32_t chunk = CHUNK_BASE, C = (R + chunk - 1) / chunk; + if (C > C_MAX) { C = C_MAX; chunk = (R + C - 1) / C; } + if (g_nosplit) { C = 1; chunk = R; } + + if (C <= 1) { + const uint32_t T = 256; + [g_enc setComputePipelineState:ctx().psoAttn]; + setInputs(); + setBufAt(out, (size_t)num_q_heads*v_hdim*2, 7); + [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; + [g_enc setBytes:&head_dim length:4 atIndex:10]; [g_enc setBytes:&v_hdim length:4 atIndex:11]; + [g_enc setBytes:&history_len length:4 atIndex:12];[g_enc setBytes:&scale length:4 atIndex:13]; + [g_enc setBytes:&kv_start length:4 atIndex:14]; [g_enc setBytes:&kv_end length:4 atIndex:15]; + [g_enc setThreadgroupMemoryLength:((size_t)kv_end + T)*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + barrier(); + return true; + } + + uint32_t TP = chunk < 128 ? chunk : 128; + TP = (TP + 31u) & ~31u; + if (TP < 32) TP = 32; if (TP > 128) TP = 128; + id partO = recycled((size_t)num_q_heads*C*v_hdim*sizeof(float)); + id partM = recycled((size_t)num_q_heads*C*sizeof(float)); + id partL = recycled((size_t)num_q_heads*C*sizeof(float)); + [g_enc setComputePipelineState:ctx().psoAttnP]; + setInputs(); + [g_enc setBuffer:partO offset:0 atIndex:7]; + [g_enc setBuffer:partM offset:0 atIndex:8]; + [g_enc setBuffer:partL offset:0 atIndex:9]; + [g_enc setBytes:&num_q_heads length:4 atIndex:10]; [g_enc setBytes:&num_kv_heads length:4 atIndex:11]; + [g_enc setBytes:&head_dim length:4 atIndex:12]; [g_enc setBytes:&v_hdim length:4 atIndex:13]; + [g_enc setBytes:&history_len length:4 atIndex:14]; [g_enc setBytes:&scale length:4 atIndex:15]; + [g_enc setBytes:&kv_start length:4 atIndex:16]; [g_enc setBytes:&kv_end length:4 atIndex:17]; + [g_enc setBytes:&chunk length:4 atIndex:18]; [g_enc setBytes:&C length:4 atIndex:19]; + [g_enc setThreadgroupMemoryLength:((size_t)chunk + TP)*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads*C,1,1) threadsPerThreadgroup:MTLSizeMake(TP,1,1)]; + barrier(); + + const uint32_t TC = 256; + [g_enc setComputePipelineState:ctx().psoAttnC]; + [g_enc setBuffer:partO offset:0 atIndex:0]; + [g_enc setBuffer:partM offset:0 atIndex:1]; + [g_enc setBuffer:partL offset:0 atIndex:2]; + setBufAt(out, (size_t)num_q_heads*v_hdim*2, 3); + [g_enc setBytes:&num_q_heads length:4 atIndex:4]; [g_enc setBytes:&v_hdim length:4 atIndex:5]; + [g_enc setBytes:&C length:4 atIndex:6]; + [g_enc setThreadgroupMemoryLength:(size_t)C*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(TC,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_attention_i8_prefill( + void* out, const void* q, const void* knew, const void* vnew, + const void* kc, const void* vc, const void* ks, const void* vs, + uint32_t num_q_heads, uint32_t num_kv_heads, uint32_t head_dim, uint32_t v_hdim, + uint32_t history_len, uint32_t new_len, uint32_t q_pos0, uint32_t window, uint32_t is_causal, uint32_t M, + float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes, + uint32_t sink, uint32_t ring) { + if (!ctx().ok) return false; + uint32_t total_keys = history_len + new_len; + const uint32_t T = 128; + if (total_keys == 0 || M == 0 || (num_q_heads % num_kv_heads) != 0) return false; + uint32_t maxsc = (ring > 0u) ? ((total_keys > sink + ring) ? (sink + ring) : total_keys) : 256u; + bool mma2 = (ring == 0u && is_causal && total_keys >= 2048u && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u); + if (mma2) { + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoAttnPreMma2]; + setBufAt(q, (size_t)M*num_q_heads*head_dim*2, 0); + if (new_len > 0 && knew && vnew) { + setBufAt(knew, (size_t)new_len*num_kv_heads*head_dim*2, 1); + setBufAt(vnew, (size_t)new_len*num_kv_heads*v_hdim*2, 2); + } else { + [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + } + auto setCacheM = [&](const void* p, size_t bytes, int idx) { + if (p && bytes) setBufAt(p, bytes, idx); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:idx]; + }; + setCacheM(kc, kc_bytes, 3); setCacheM(vc, vc_bytes, 4); setCacheM(ks, ks_bytes, 5); setCacheM(vs, vs_bytes, 6); + setBufAt(out, (size_t)M*num_q_heads*v_hdim*2, 7); + [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; + [g_enc setBytes:&head_dim length:4 atIndex:10]; [g_enc setBytes:&v_hdim length:4 atIndex:11]; + [g_enc setBytes:&history_len length:4 atIndex:12];[g_enc setBytes:&scale length:4 atIndex:13]; + [g_enc setBytes:&q_pos0 length:4 atIndex:14]; [g_enc setBytes:&new_len length:4 atIndex:15]; + [g_enc setBytes:&M length:4 atIndex:16]; + uint32_t mtiles = (M + 7u)/8u; + [g_enc dispatchThreadgroups:MTLSizeMake(mtiles, 1, 1) threadsPerThreadgroup:MTLSizeMake(512,1,1)]; + barrier(); + return true; + } + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoAttnPre]; + setBufAt(q, (size_t)M*num_q_heads*head_dim*2, 0); + if (new_len > 0 && knew && vnew) { + setBufAt(knew, (size_t)new_len*num_kv_heads*head_dim*2, 1); + setBufAt(vnew, (size_t)new_len*num_kv_heads*v_hdim*2, 2); + } else { + [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + } + auto setCache = [&](const void* p, size_t bytes, int idx) { + if (p && bytes) setBufAt(p, bytes, idx); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:idx]; + }; + setCache(kc, kc_bytes, 3); setCache(vc, vc_bytes, 4); setCache(ks, ks_bytes, 5); setCache(vs, vs_bytes, 6); + setBufAt(out, (size_t)M*num_q_heads*v_hdim*2, 7); + [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; + [g_enc setBytes:&head_dim length:4 atIndex:10]; [g_enc setBytes:&v_hdim length:4 atIndex:11]; + [g_enc setBytes:&history_len length:4 atIndex:12];[g_enc setBytes:&scale length:4 atIndex:13]; + [g_enc setBytes:&q_pos0 length:4 atIndex:14]; [g_enc setBytes:&new_len length:4 atIndex:15]; + [g_enc setBytes:&window length:4 atIndex:16]; [g_enc setBytes:&is_causal length:4 atIndex:17]; + [g_enc setBytes:&maxsc length:4 atIndex:18]; + [g_enc setBytes:&sink length:4 atIndex:19]; [g_enc setBytes:&ring length:4 atIndex:20]; + [g_enc setThreadgroupMemoryLength:((size_t)maxsc + T)*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(M*num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* oshape, + const uint32_t* sstride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes) { + if (!ctx().ok || ndim == 0 || ndim > 8) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoStrided]; + setBufAt(in, in_bytes, 0); setBufAt(out, out_bytes, 1); + [g_enc setBytes:oshape length:ndim*4 atIndex:2]; [g_enc setBytes:sstride length:ndim*4 atIndex:3]; + [g_enc setBytes:&ndim length:4 atIndex:4]; [g_enc setBytes:&total length:4 atIndex:5]; [g_enc setBytes:&base length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_bcast_binary(int op, void* out, const void* a, const void* b, + const uint32_t* oshape, const uint32_t* astride, const uint32_t* bstride, uint32_t ndim, uint32_t total, + size_t a_bytes, size_t b_bytes, size_t out_bytes) { + if (!ctx().ok || ndim == 0 || ndim > 8) return false; + ensureEncoder(); + int o = op; + [g_enc setComputePipelineState:ctx().psoBcast]; + setBufAt(a, a_bytes, 0); setBufAt(b, b_bytes, 1); setBufAt(out, out_bytes, 2); + [g_enc setBytes:oshape length:ndim*4 atIndex:3]; [g_enc setBytes:astride length:ndim*4 atIndex:4]; [g_enc setBytes:bstride length:ndim*4 atIndex:5]; + [g_enc setBytes:&ndim length:4 atIndex:6]; [g_enc setBytes:&total length:4 atIndex:7]; [g_enc setBytes:&o length:4 atIndex:8]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_strided_scatter(void* out, const void* in, const uint32_t* ishape, + const uint32_t* ostride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes) { + if (!ctx().ok || ndim == 0 || ndim > 8) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoScatter]; + setBufAt(in, in_bytes, 0); setBufAt(out, out_bytes, 1); + [g_enc setBytes:ishape length:ndim*4 atIndex:2]; [g_enc setBytes:ostride length:ndim*4 atIndex:3]; + [g_enc setBytes:&ndim length:4 atIndex:4]; [g_enc setBytes:&total length:4 atIndex:5]; [g_enc setBytes:&base length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} +bool cactus_metal_encode_kv_append_i8(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, + size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + if (!ctx().ok) return false; + uint32_t num_groups = (hdim + group_size - 1)/group_size, n = kv_heads*num_groups; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoKvAppend]; + setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); + [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; + [g_enc setBytes:¤t_len length:4 atIndex:5]; [g_enc setBytes:&group_size length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_kv_append_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, + size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + if (!ctx().ok) return false; + uint32_t num_groups = (hdim + group_size - 1)/group_size, n = M*kv_heads*num_groups; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoKvAppendM]; + setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); + [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; + [g_enc setBytes:¤t_len length:4 atIndex:5]; [g_enc setBytes:&group_size length:4 atIndex:6]; + [g_enc setBytes:&M length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, + uint32_t sink, uint32_t W, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + if (!ctx().ok) return false; + uint32_t num_groups = (hdim + group_size - 1)/group_size, n = M*kv_heads*num_groups; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoKvAppendRingM]; + setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); + [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; + [g_enc setBytes:¤t_len length:4 atIndex:5]; [g_enc setBytes:&group_size length:4 atIndex:6]; + [g_enc setBytes:&M length:4 atIndex:7]; + [g_enc setBytes:&sink length:4 atIndex:8]; [g_enc setBytes:&W length:4 atIndex:9]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, + uint32_t group_size, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + if (!ctx().ok) return false; + uint32_t num_groups = (hdim + group_size - 1)/group_size, per = kv_heads*num_groups; + id scrI8 = recycled((size_t)(remaining?remaining:1)*kv_heads*hdim); + id scrSc = recycled((size_t)(remaining?remaining:1)*kv_heads*num_groups*sizeof(float)); + ensureEncoder(); + if (remaining > 0) { + uint32_t n = remaining*per; + [g_enc setComputePipelineState:ctx().psoSlideS]; + setBufAt(int8base, int8_bytes, 0); setBufAt(scalebase, scale_bytes, 1); + [g_enc setBuffer:scrI8 offset:0 atIndex:2]; [g_enc setBuffer:scrSc offset:0 atIndex:3]; + [g_enc setBytes:&kv_heads length:4 atIndex:4]; [g_enc setBytes:&hdim length:4 atIndex:5]; + [g_enc setBytes:&group_size length:4 atIndex:6]; [g_enc setBytes:&shift_src length:4 atIndex:7]; + [g_enc setBytes:&remaining length:4 atIndex:8]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; + barrier(); + } + uint32_t n2 = (remaining+1)*per; + [g_enc setComputePipelineState:ctx().psoSlideR]; + setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); + [g_enc setBuffer:scrI8 offset:0 atIndex:3]; [g_enc setBuffer:scrSc offset:0 atIndex:4]; + [g_enc setBytes:&kv_heads length:4 atIndex:5]; [g_enc setBytes:&hdim length:4 atIndex:6]; + [g_enc setBytes:&group_size length:4 atIndex:7]; [g_enc setBytes:&keep_sink length:4 atIndex:8]; + [g_enc setBytes:&remaining length:4 atIndex:9]; + [g_enc dispatchThreads:MTLSizeMake(n2,1,1) threadsPerThreadgroup:MTLSizeMake(n2<256?n2:256,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, + uint32_t group_size, uint32_t M, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + if (!ctx().ok) return false; + uint32_t num_groups = (hdim + group_size - 1)/group_size, per = kv_heads*num_groups; + id scrI8 = recycled((size_t)(remaining?remaining:1)*kv_heads*hdim); + id scrSc = recycled((size_t)(remaining?remaining:1)*kv_heads*num_groups*sizeof(float)); + ensureEncoder(); + if (remaining > 0) { + uint32_t n = remaining*per; + [g_enc setComputePipelineState:ctx().psoSlideS]; + setBufAt(int8base, int8_bytes, 0); setBufAt(scalebase, scale_bytes, 1); + [g_enc setBuffer:scrI8 offset:0 atIndex:2]; [g_enc setBuffer:scrSc offset:0 atIndex:3]; + [g_enc setBytes:&kv_heads length:4 atIndex:4]; [g_enc setBytes:&hdim length:4 atIndex:5]; + [g_enc setBytes:&group_size length:4 atIndex:6]; [g_enc setBytes:&shift_src length:4 atIndex:7]; + [g_enc setBytes:&remaining length:4 atIndex:8]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; + barrier(); + } + uint32_t n2 = (remaining+M)*per; + [g_enc setComputePipelineState:ctx().psoSlideRM]; + setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); + [g_enc setBuffer:scrI8 offset:0 atIndex:3]; [g_enc setBuffer:scrSc offset:0 atIndex:4]; + [g_enc setBytes:&kv_heads length:4 atIndex:5]; [g_enc setBytes:&hdim length:4 atIndex:6]; + [g_enc setBytes:&group_size length:4 atIndex:7]; [g_enc setBytes:&keep_sink length:4 atIndex:8]; + [g_enc setBytes:&remaining length:4 atIndex:9]; [g_enc setBytes:&M length:4 atIndex:10]; + [g_enc dispatchThreads:MTLSizeMake(n2,1,1) threadsPerThreadgroup:MTLSizeMake(n2<256?n2:256,1,1)]; + barrier(); + return true; +} + +void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, + uint32_t M, __fp16* C) { + const uint32_t gs = W->group_size; + const bool fast = + ctx().ok && M == 1 && W->bits == 4 && gs >= 128 && (gs % 128) == 0 && + !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && (W->N % 4) == 0 && + W->input_scale_recip != nullptr && W->left_signs != nullptr && + W->right_signs != nullptr && W->permutation != nullptr && + W->codebook != nullptr && W->norms != nullptr && W->packed_indices != nullptr; + + if (getenv("CACTUS_GPU_DEBUG") && M == 1) { + static std::atomic dbg{0}; + if (dbg++ < 14) + fprintf(stderr, "[metal] M=%u bits=%u gs=%u flags=0x%x recip=%d ls=%d rs=%d perm=%d -> %s\n", + M, W->bits, gs, W->flags, W->input_scale_recip != nullptr, + W->left_signs != nullptr, W->right_signs != nullptr, + W->permutation != nullptr, fast ? "GPU" : "CPU-fallback"); + } + + if (!fast) { cactus_quant_matmul(W, A, M, C); return; } + + @autoreleasepool { + const uint32_t K = W->K, N = W->N, ng = W->num_groups; + const uint32_t pgb = (gs * 4u + 7u) / 8u; + + ResW& rw = resident(W); + id bx = buf(A, (size_t)K * sizeof(__fp16)); + id brecip = rw.recip; + id bls = rw.lsign; + id brs = rw.rsign; + id bperm = rw.perm; + id bpk = rw.packed; + id bcb = rw.codebook; + id bnorm = rw.norms; + id bcode = [ctx().dev newBufferWithLength:(size_t)ng * gs * sizeof(__fp16) + options:MTLResourceStorageModeShared]; + id by = [ctx().dev newBufferWithLength:(size_t)N * sizeof(__fp16) + options:MTLResourceStorageModeShared]; + + id cmd = [ctx().queue commandBuffer]; + id e = [cmd computeCommandEncoder]; + + bool simdT = (gs==128u && ctx().psoTsimd); + [e setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; + [e setBuffer:bx offset:0 atIndex:0]; [e setBuffer:brecip offset:0 atIndex:1]; + [e setBuffer:bls offset:0 atIndex:2]; [e setBuffer:brs offset:0 atIndex:3]; + [e setBuffer:bperm offset:0 atIndex:4]; [e setBuffer:bcode offset:0 atIndex:5]; + [e setBytes:&gs length:sizeof(uint32_t) atIndex:6]; + [e setThreadgroupMemoryLength:gs * sizeof(float) atIndex:0]; + [e dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; + + [e setComputePipelineState:ctx().psoG]; + [e setBuffer:bcode offset:0 atIndex:0]; [e setBuffer:bpk offset:0 atIndex:1]; + [e setBuffer:bcb offset:0 atIndex:2]; [e setBuffer:bnorm offset:0 atIndex:3]; + [e setBuffer:by offset:0 atIndex:4]; + [e setBytes:&gs length:sizeof(uint32_t) atIndex:5]; + [e setBytes:&ng length:sizeof(uint32_t) atIndex:6]; + [e setBytes:&pgb length:sizeof(uint32_t) atIndex:7]; + [e setBytes:&N length:sizeof(uint32_t) atIndex:8]; + const uint32_t ROWS = 8; + [e dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) + threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + + [e endEncoding]; + [cmd commit]; + [cmd waitUntilCompleted]; + + std::memcpy(C, [by contents], (size_t)N * sizeof(__fp16)); + } +} + +#endif diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp new file mode 100644 index 000000000..778f4a976 --- /dev/null +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -0,0 +1,58 @@ + +#include "metal_backend.h" + +#if !CACTUS_HAS_METAL +bool cactus_metal_available() { return false; } +void cactus_metal_set_active(bool) {} +bool cactus_metal_active_mode() { return false; } +bool cactus_metal_concurrent() { return false; } +void cactus_metal_barrier() {} +void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, + uint32_t M, __fp16* C) { + cactus_quant_matmul(W, A, M, C); +} +void cactus_metal_session_begin() {} +void cactus_metal_session_sync() {} +void cactus_metal_session_end() {} +void* cactus_metal_alloc_shared(size_t) { return nullptr; } +void cactus_metal_free_shared(void*) {} +bool cactus_metal_encode_copy(void*, const void*, size_t) { return false; } +bool cactus_metal_encode_binary(int, void*, const void*, const void*, size_t) { return false; } +bool cactus_metal_encode_scalar(int, void*, const void*, size_t, float) { return false; } +bool cactus_metal_encode_unary(int, void*, const void*, size_t) { return false; } +bool cactus_metal_encode_swiglu(void*, const void*, const void*, size_t, float) { return false; } +bool cactus_metal_encode_rope(void*, const void*, const void*, const void*, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_rms_norm(void*, const void*, const void*, size_t, size_t, float) { return false; } +bool cactus_metal_encode_rms_norm_add(void*, const void*, const void*, const void*, size_t, size_t, float) { return false; } +bool cactus_metal_encode_argmax(const void*, uint32_t, void*) { return false; } +bool cactus_metal_encode_cast(void*, int, const void*, int, size_t) { return false; } +bool cactus_metal_encode_quant_matmul(void*, const void*, const CactusQuantMatrix*) { return false; } +bool cactus_metal_encode_quant_matmul_m(void*, const void*, const CactusQuantMatrix*, uint32_t) { return false; } +bool cactus_metal_prewarm_quant(const CactusQuantMatrix*) { return false; } +bool cactus_metal_encode_quant_matmul_ortho(void*, const void*, void*, const CactusQuantMatrix*) { return false; } +bool cactus_metal_encode_embedding_ortho(void*, uint32_t, const CactusQuantMatrix*) { return false; } +bool cactus_metal_encode_embedding_hadamard(void*, uint32_t, const CactusQuantMatrix*) { return false; } +bool cactus_metal_encode_embedding_ortho_m(void*, const CactusQuantMatrix*, const uint32_t*, uint32_t) { return false; } +bool cactus_metal_encode_embedding_hadamard_m(void*, const CactusQuantMatrix*, const uint32_t*, uint32_t) { return false; } +bool cactus_metal_encode_gather_f16(void*, const void*, size_t, const uint32_t*, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_attention_i8(void*, const void*, const void*, const void*, const void*, const void*, + const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, + float, size_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_attention_i8_prefill(void*, const void*, const void*, const void*, const void*, const void*, + const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, float, size_t, size_t, size_t, size_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_strided_copy(void*, const void*, const uint32_t*, const uint32_t*, + uint32_t, uint32_t, uint32_t, size_t, size_t) { return false; } +bool cactus_metal_encode_bcast_binary(int, void*, const void*, const void*, const uint32_t*, + const uint32_t*, const uint32_t*, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_strided_scatter(void*, const void*, const uint32_t*, const uint32_t*, + uint32_t, uint32_t, uint32_t, size_t, size_t) { return false; } +bool cactus_metal_encode_kv_append_i8(const void*, void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_kv_append_sliding_i8(const void*, void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_kv_append_i8_m(const void*, void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_kv_append_ring_i8_m(const void*, void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +#endif diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 0e01978b7..28a701a89 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -8,7 +8,10 @@ DEFAULT_TEST_MODEL_ID, DEFAULT_TEST_TRANSCRIPTION_MODEL_ID, SUPPORTED_PLATFORMS, + default_backend, ) + +_BACKEND_HELP = "Inference backend: metal (fused Apple GPU / Metal path) or cpu. Defaults to metal when the device supports it." from .download import cmd_download _PLATFORM_CHOICES = ("auto", "cpu", *SUPPORTED_PLATFORMS) @@ -285,6 +288,8 @@ def create_parser(): help="Confidence threshold below which local completions may hand off to cloud") run_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, help="Maximum time to wait for cloud handoff before falling back locally") + run_parser.add_argument("--backend", choices=["cpu", "metal"], default=default_backend(), + help=_BACKEND_HELP) transcribe_parser = subparsers.add_parser("transcribe", help="Transcribe audio with a model", parents=[_telemetry_parent(), _build_parent()]) @@ -295,6 +300,8 @@ def create_parser(): help="Audio file to transcribe (WAV)") transcribe_parser.add_argument("--language", default="en", help="Language code (default: en)") + transcribe_parser.add_argument("--backend", choices=["cpu", "metal"], default=default_backend(), + help=_BACKEND_HELP) serve_parser = subparsers.add_parser("serve", help="OpenAI-compatible local HTTP server", parents=[_telemetry_parent(), _build_parent()]) @@ -313,6 +320,8 @@ def create_parser(): help="Maximum time to wait for cloud handoff before falling back locally") serve_parser.add_argument("--no-access-log", action="store_true", help="Disable per-request HTTP access logging") + serve_parser.add_argument("--backend", choices=["cpu", "metal"], default=default_backend(), + help=_BACKEND_HELP) code_parser = subparsers.add_parser("code", help="Run the Cactus coding agent (TUI / print mode)", parents=[_build_parent()]) @@ -330,6 +339,8 @@ def create_parser(): help="Confidence threshold below which completions hand off to cloud") code_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, help="Maximum time to wait for cloud handoff before falling back locally") + code_parser.add_argument("--backend", choices=["cpu", "metal"], default=default_backend(), + help=_BACKEND_HELP) code_parser.add_argument("agent_args", nargs=argparse.REMAINDER, help="Arguments passed through to the coding agent (prefix with -- )") diff --git a/python/cactus/cli/code.py b/python/cactus/cli/code.py index 87d023a20..e563412a9 100644 --- a/python/cactus/cli/code.py +++ b/python/cactus/cli/code.py @@ -7,7 +7,7 @@ import urllib.request from pathlib import Path -from .common import BLUE, DEFAULT_MODEL_ID, GREEN, RED, YELLOW, is_valid_bundle, print_color, weights_root +from .common import BLUE, DEFAULT_MODEL_ID, GREEN, RED, YELLOW, default_backend, is_valid_bundle, print_color, weights_root _PACKAGE_DIR = Path(__file__).resolve().parent.parent _PROJECT_ROOT = _PACKAGE_DIR.parent.parent @@ -147,6 +147,7 @@ def cmd_code(args) -> int: "--host", args.host, "--port", str(args.port), "--no-access-log", "--bits", str(getattr(args, "bits", 4)), "--platform", getattr(args, "platform", "auto"), + "--backend", getattr(args, "backend", default_backend()), ] if getattr(args, "token", None): serve_cmd += ["--token", args.token] diff --git a/python/cactus/cli/common.py b/python/cactus/cli/common.py index 71601e4cb..370075206 100644 --- a/python/cactus/cli/common.py +++ b/python/cactus/cli/common.py @@ -1,9 +1,15 @@ #!/usr/bin/env python3 import os +import platform import subprocess import sys from pathlib import Path + +def default_backend() -> str: + """Prefer the Metal GPU backend on Apple hardware, else CPU.""" + return "metal" if platform.system() == "Darwin" else "cpu" + SCRIPT_DIR = Path(__file__).resolve().parent @@ -86,6 +92,12 @@ def apply_runtime_env(args) -> None: and load the stored cloud API key.""" if getattr(args, "no_cloud_tele", False): os.environ["CACTUS_NO_CLOUD_TELE"] = "1" + backend = getattr(args, "backend", None) + if backend == "metal": + os.environ["CACTUS_GPU_FUSED"] = "1" + elif backend == "cpu": + os.environ.pop("CACTUS_GPU_FUSED", None) + os.environ.pop("CACTUS_GPU", None) apply_cloud_api_key_env() diff --git a/python/cactus/cli/compile.py b/python/cactus/cli/compile.py index c5193cd06..0bc7ef1b7 100644 --- a/python/cactus/cli/compile.py +++ b/python/cactus/cli/compile.py @@ -99,6 +99,7 @@ def build_binary( "-framework", "Accelerate", "-framework", "CoreML", "-framework", "Foundation", + "-framework", "Metal", "-framework", "Security", "-framework", "SystemConfiguration", "-framework", "CFNetwork", diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index e7bd3814a..38078a3a8 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -40,6 +40,7 @@ def cmd_run(args) -> int: ("--result-json", args.result_json), ("--confidence-threshold", args.confidence_threshold), ("--cloud-timeout-ms", args.cloud_timeout_ms), + ("--backend", args.backend), ): if value is not None: cmd.extend([flag, str(value)]) From e6934b2846789cba222bd8e3295a921a31ffc179 Mon Sep 17 00:00:00 2001 From: jakmro Date: Wed, 1 Jul 2026 14:07:05 +0200 Subject: [PATCH 02/36] cleanup Signed-off-by: jakmro --- README.md | 4 ++++ cactus-engine/cactus_engine.h | 7 ++----- cactus-engine/src/complete.cpp | 2 -- cactus-engine/src/init.cpp | 3 +-- cactus-engine/src/model.cpp | 2 +- cactus-engine/tests/run.cpp | 14 +++++++------- cactus-graph/cactus_graph.h | 11 +++-------- cactus-graph/src/execute.cpp | 29 +++++++++++++---------------- cactus-graph/src/gpu_fused.cpp | 3 --- cactus-graph/src/ops_cache.cpp | 4 ++-- cactus-graph/src/ops_math.cpp | 5 +++-- cactus-graph/src/ops_nn.cpp | 9 +++------ cactus-graph/src/param_io.cpp | 2 +- python/cactus/bindings/cactus.py | 8 ++++++++ python/cactus/cli/__init__.py | 19 ++++++++----------- python/cactus/cli/code.py | 4 ++-- python/cactus/cli/common.py | 12 ------------ 17 files changed, 58 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 327c8d5bc..bf37101a4 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ graph.hard_reset(); │ cactus run [model|path] run a model (downloads if needed) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ │ --platform auto|cpu|apple target platform (default: auto) │ +│ --backend auto|cpu|metal inference backend (default: auto) │ │ --image image file for VLM inference │ │ --audio audio file for audio chat │ │ --system system prompt │ @@ -210,6 +211,7 @@ graph.hard_reset(); │ --language language code (default: en) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ │ --platform auto|cpu|apple target platform (default: auto) │ +│ --backend auto|cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ │ @@ -234,6 +236,7 @@ graph.hard_reset(); │ --port port (default: 8080) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ │ --platform auto|cpu|apple target platform (default: auto) │ +│ --backend auto|cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ --no-cloud-handoff disable automatic cloud handoff │ @@ -244,6 +247,7 @@ graph.hard_reset(); │ --serve-model auto-start a server with this model │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ │ --platform auto|cpu|apple target platform (default: auto) │ +│ --backend auto|cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ --host server address (default: 127.0.0.1) │ diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index e3f40c5b2..18ee07012 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -34,11 +34,8 @@ CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); -// Backend selection. "metal" runs the Apple GPU / Metal fused path (when available), "cpu" the CPU path. -// Overrides the CACTUS_GPU[_FUSED] env default. Call before cactus_init / the first forward. -CACTUS_FFI_EXPORT void cactus_set_backend(const char* backend); -CACTUS_FFI_EXPORT const char* cactus_get_backend(void); // resolved: "metal" or "cpu" -CACTUS_FFI_EXPORT bool cactus_metal_available(void); // true if a Metal device is present +// backend: "auto", "cpu", or "metal". Returns 0 on success, -1 on failure +CACTUS_FFI_EXPORT int cactus_set_backend(const char* backend); CACTUS_FFI_EXPORT int cactus_complete( cactus_model_t model, diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index c5e732f25..51ef39b06 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -816,7 +816,6 @@ int cactus_complete( CACTUS_LOG_DEBUG("complete", "Prompt tokens: " << prompt.tokens.size() << ", max_tokens: " << prompt.options.max_tokens); - g_cactus_force_cpu = false; bool has_images = prompt.has_images(); bool has_audio = prompt.has_audio(); @@ -1429,7 +1428,6 @@ int cactus_benchmark_tokens( peak_ram_usage_mb = std::max(peak_ram_usage_mb, get_ram_usage_mb()); }; handle->model->reset_cache(); - g_cactus_force_cpu = false; auto prefill_start_time = std::chrono::high_resolution_clock::now(); size_t cache_prime_tokens = 0; uint32_t next_token = 0; diff --git a/cactus-engine/src/init.cpp b/cactus-engine/src/init.cpp index b5f2f55e9..ca11df7fa 100644 --- a/cactus-engine/src/init.cpp +++ b/cactus-engine/src/init.cpp @@ -349,8 +349,7 @@ bool matches_stop_sequence(const std::vector& generated_tokens, extern "C" { -void cactus_set_backend(const char* backend) { cactus_backend_select(backend); } -const char* cactus_get_backend(void) { return cactus_backend_name(); } +int cactus_set_backend(const char* backend) { return cactus_backend_select(backend); } cactus_model_t cactus_init(const char* model_path, const char* corpus_dir, bool cache_index) { constexpr size_t DEFAULT_CONTEXT_SIZE = 512; // matches default sliding window size diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 327ccdf97..5381569be 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -891,7 +891,7 @@ void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { const bool fused = cactus_backend_fused(); static const bool nofold = std::getenv("CACTUS_FUSED_NOFOLD") != nullptr; if (fused && !nofold) { - static int state = 0; static FusedEmbedCtx ctx; // 0=unknown 1=ok 2=unsupported + static int state = 0; static FusedEmbedCtx ctx; if (state == 0) state = encoder_->graph->extract_ple_pathway(ctx) ? 1 : 2; if (state == 1) { ctx.token_id = static_cast(token_id); ctx.position = static_cast(position); diff --git a/cactus-engine/tests/run.cpp b/cactus-engine/tests/run.cpp index f07e1bde6..58d47ce67 100644 --- a/cactus-engine/tests/run.cpp +++ b/cactus-engine/tests/run.cpp @@ -855,13 +855,13 @@ int main(int argc, char** argv) { } } - // --backend metal enables the fused Metal GPU path, gated by CACTUS_GPU_FUSED in cactus-graph - // (read once at the first forward, so it must be set before cactus_init). Default is CPU. - if (backend == "metal" || backend == "gpu") { - setenv("CACTUS_GPU_FUSED", "1", 1); - std::cout << "Backend: Metal GPU\n"; - } else if (!backend.empty() && backend != "cpu") { - std::cerr << "Error: unknown backend '" << backend << "' (expected 'cpu' or 'metal')\n"; + if (backend == "cpu" || backend == "metal") { + if (cactus_set_backend(backend.c_str()) == 0) + std::cout << "Backend: " << (backend == "metal" ? "Metal GPU" : "CPU") << "\n"; + else + std::cout << "Metal not available; using CPU\n"; + } else if (!backend.empty() && backend != "auto") { + std::cerr << "Error: unknown backend '" << backend << "' (expected 'cpu', 'metal', or 'auto')\n"; return 1; } diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 4641e3359..22c553f3b 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -17,12 +17,9 @@ #include #include -// Backend selection: "metal" or "cpu". Overrides the CACTUS_GPU[_FUSED] env default. -// cactus_backend_metal() gates the GPU op path; cactus_backend_fused() gates the fully-fused decode. -void cactus_backend_select(const char* backend); +int cactus_backend_select(const char* backend); bool cactus_backend_metal(); bool cactus_backend_fused(); -const char* cactus_backend_name(); namespace cactus { @@ -88,7 +85,7 @@ class Logger { #define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) #define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) -enum class ComputeBackend { CPU, NPU, GPU }; +enum class ComputeBackend { CPU, NPU }; enum class Activation { SILU, GELU, GELU_ERF, RELU, SIGMOID, TANH }; @@ -464,15 +461,13 @@ struct FusedEmbedCtx { CactusQuantMatrix ple{}; CactusQuantMatrix proj{}; const void* rms_weight = nullptr; - float emb_scale = 0, ple_scale = 0, proj_scale = 0, final_scale = 0, rms_eps = 1e-6f; + float emb_scale = 0.0f, ple_scale = 0.0f, proj_scale = 0.0f, final_scale = 0.0f, rms_eps = 1e-6f; bool ok = false; }; void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx); bool cactus_graph_gpu_argmax(uint32_t* idx, float* best, float* second); -extern bool g_cactus_force_cpu; - class CactusGraph { public: CactusGraph(); diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index d3a7b73a9..031fd3ea0 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -15,23 +15,20 @@ #include #include -static int g_backend_override = -1; // -1 = env default, 0 = cpu, 1 = metal -void cactus_backend_select(const char* backend) { - if (!backend) { g_backend_override = -1; return; } - if (std::strcmp(backend, "metal") == 0) g_backend_override = 1; - else if (std::strcmp(backend, "cpu") == 0) g_backend_override = 0; -} -bool cactus_backend_metal() { - if (g_backend_override == 0) return false; - if (g_backend_override == 1) return cactus_metal_available(); - return (std::getenv("CACTUS_GPU") != nullptr || std::getenv("CACTUS_GPU_FUSED") != nullptr) && cactus_metal_available(); -} -bool cactus_backend_fused() { - if (g_backend_override == 0) return false; - if (g_backend_override == 1) return cactus_metal_available(); - return std::getenv("CACTUS_GPU_FUSED") != nullptr && cactus_metal_available(); +static int g_backend_override = -1; +int cactus_backend_select(const char* backend) { + if (!backend) return -1; + if (std::strcmp(backend, "auto") == 0) { g_backend_override = -1; return 0; } + if (std::strcmp(backend, "metal") == 0) { + if (!cactus_metal_available()) return -1; + g_backend_override = 1; + return 0; + } + if (std::strcmp(backend, "cpu") == 0) { g_backend_override = 0; return 0; } + return -1; } -const char* cactus_backend_name() { return cactus_backend_metal() ? "metal" : "cpu"; } +bool cactus_backend_metal() { return g_backend_override != 0 && cactus_metal_available(); } +bool cactus_backend_fused() { return g_backend_override != 0 && cactus_metal_available(); } using ComputeFn = void(*)(GraphNode&, const nodes_vector&, const node_index_map_t&); diff --git a/cactus-graph/src/gpu_fused.cpp b/cactus-graph/src/gpu_fused.cpp index fa800738d..cb3a06bc6 100644 --- a/cactus-graph/src/gpu_fused.cpp +++ b/cactus-graph/src/gpu_fused.cpp @@ -60,8 +60,6 @@ static const float* g_gpu_argmax_buf = nullptr; } -bool g_cactus_force_cpu = false; - bool cactus_graph_gpu_argmax(uint32_t* idx, float* best, float* second) { if (!g_gpu_argmax_valid || !g_gpu_argmax_buf) return false; g_gpu_argmax_valid = false; @@ -102,7 +100,6 @@ bool CactusGraph::extract_ple_pathway(FusedEmbedCtx& ctx) const { bool CactusGraph::execute_gpu_fused() { static const bool dbg = std::getenv("CACTUS_FUSED_DEBUG") != nullptr; if (!cactus_metal_available()) return false; - if (g_cactus_force_cpu) return false; const size_t n = nodes_.size(); if (n < 3861) return false; auto B = [&](size_t idx) -> BufferDesc& { return nodes_[idx]->output_buffer; }; diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 5e7f33097..6fbadf68a 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -145,7 +145,7 @@ inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { } get_meta(resized)->max_seq_len = new_max; buf = std::move(resized); - if (shared) cactus_metal_free_shared(old_data); // reclaim the old shared buffer (no-op if not shared) + if (shared) cactus_metal_free_shared(old_data); return true; } @@ -189,7 +189,7 @@ void compute_kv_cache_state_node( : cache_buffer_size(max_seq, kv_heads, hdim); node.output_buffer = BufferDesc({num_slots * per_slot}, fp16_cache ? Precision::FP16 : Precision::INT8); - if (kv_cache_resident()) { // GPU-resident shared cache (no snapshots) + if (kv_cache_resident()) { void* p = cactus_metal_alloc_shared(node.output_buffer.byte_size); if (p) node.output_buffer.set_external(p); else node.output_buffer.allocate(); } else { diff --git a/cactus-graph/src/ops_math.cpp b/cactus-graph/src/ops_math.cpp index 8ac6e3d0a..771bee1dd 100644 --- a/cactus-graph/src/ops_math.cpp +++ b/cactus-graph/src/ops_math.cpp @@ -609,10 +609,11 @@ void compute_precision_cast_node(GraphNode& node, const std::vector alias + if (cactus_metal_active_mode()) { node.output_buffer.set_external(const_cast(input_buf.get_data())); - else + } else { std::memcpy(node.output_buffer.get_data(), input_buf.get_data(), input_buf.byte_size); + } return; } diff --git a/cactus-graph/src/ops_nn.cpp b/cactus-graph/src/ops_nn.cpp index 50d7c6029..3582c5f5a 100644 --- a/cactus-graph/src/ops_nn.cpp +++ b/cactus-graph/src/ops_nn.cpp @@ -56,13 +56,10 @@ void compute_matmul_node(GraphNode& node, const std::vector(M), output); - } else if (is_orthogonal) { + if (is_orthogonal) { cactus_quant_orthogonal_matmul(&mat, lhs, static_cast(M), output); + } else if (cactus_backend_metal()) { + cactus_metal_quant_matmul(&mat, lhs, static_cast(M), output); } else { cactus_quant_matmul(&mat, lhs, static_cast(M), output); } diff --git a/cactus-graph/src/param_io.cpp b/cactus-graph/src/param_io.cpp index 7d648f269..2e3fdadd4 100644 --- a/cactus-graph/src/param_io.cpp +++ b/cactus-graph/src/param_io.cpp @@ -314,7 +314,7 @@ void read_field(std::istream& in, ParamField field, OpParams& params) { case ParamField::PretransposedRhs: params.pretransposed_rhs = (read_u32(in) != 0); break; case ParamField::Backend: { uint32_t backend_val = read_u32(in); - if (backend_val > static_cast(ComputeBackend::GPU)) { + if (backend_val > static_cast(ComputeBackend::NPU)) { throw std::runtime_error("Graph file corrupted: invalid backend"); } params.backend = static_cast(backend_val); diff --git a/python/cactus/bindings/cactus.py b/python/cactus/bindings/cactus.py index 27e85f84d..b575f85b6 100644 --- a/python/cactus/bindings/cactus.py +++ b/python/cactus/bindings/cactus.py @@ -67,6 +67,9 @@ class cactus_tensor_info_t(ctypes.Structure): _lib.cactus_init.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_bool] _lib.cactus_init.restype = ctypes.c_void_p +_lib.cactus_set_backend.argtypes = [ctypes.c_char_p] +_lib.cactus_set_backend.restype = ctypes.c_int + # cactus graph API _lib.cactus_graph_create.restype = cactus_graph_t _lib.cactus_graph_destroy.argtypes = [cactus_graph_t] @@ -952,6 +955,11 @@ def cactus_telemetry_shutdown(): # ── Model lifecycle ────────────────────────────────────────────────── +def cactus_set_backend(backend): + """Select the inference backend ("auto", "cpu", or "metal"). Returns 0 on success, -1 on failure.""" + return _lib.cactus_set_backend(_enc(backend)) + + def cactus_init(model_path, corpus_dir=None, cache_index=False): """Load a model from disk. diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 28a701a89..f0e2e4174 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -8,10 +8,7 @@ DEFAULT_TEST_MODEL_ID, DEFAULT_TEST_TRANSCRIPTION_MODEL_ID, SUPPORTED_PLATFORMS, - default_backend, ) - -_BACKEND_HELP = "Inference backend: metal (fused Apple GPU / Metal path) or cpu. Defaults to metal when the device supports it." from .download import cmd_download _PLATFORM_CHOICES = ("auto", "cpu", *SUPPORTED_PLATFORMS) @@ -288,8 +285,8 @@ def create_parser(): help="Confidence threshold below which local completions may hand off to cloud") run_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, help="Maximum time to wait for cloud handoff before falling back locally") - run_parser.add_argument("--backend", choices=["cpu", "metal"], default=default_backend(), - help=_BACKEND_HELP) + run_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", + help="Inference backend, default auto (Metal on Apple GPU, gemma-4-E2B-it only)") transcribe_parser = subparsers.add_parser("transcribe", help="Transcribe audio with a model", parents=[_telemetry_parent(), _build_parent()]) @@ -300,8 +297,8 @@ def create_parser(): help="Audio file to transcribe (WAV)") transcribe_parser.add_argument("--language", default="en", help="Language code (default: en)") - transcribe_parser.add_argument("--backend", choices=["cpu", "metal"], default=default_backend(), - help=_BACKEND_HELP) + transcribe_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", + help="Inference backend, default auto (Metal on Apple GPU, gemma-4-E2B-it only)") serve_parser = subparsers.add_parser("serve", help="OpenAI-compatible local HTTP server", parents=[_telemetry_parent(), _build_parent()]) @@ -320,8 +317,8 @@ def create_parser(): help="Maximum time to wait for cloud handoff before falling back locally") serve_parser.add_argument("--no-access-log", action="store_true", help="Disable per-request HTTP access logging") - serve_parser.add_argument("--backend", choices=["cpu", "metal"], default=default_backend(), - help=_BACKEND_HELP) + serve_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", + help="Inference backend, default auto (Metal on Apple GPU, gemma-4-E2B-it only)") code_parser = subparsers.add_parser("code", help="Run the Cactus coding agent (TUI / print mode)", parents=[_build_parent()]) @@ -339,8 +336,8 @@ def create_parser(): help="Confidence threshold below which completions hand off to cloud") code_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, help="Maximum time to wait for cloud handoff before falling back locally") - code_parser.add_argument("--backend", choices=["cpu", "metal"], default=default_backend(), - help=_BACKEND_HELP) + code_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", + help="Inference backend, default auto (Metal on Apple GPU, gemma-4-E2B-it only)") code_parser.add_argument("agent_args", nargs=argparse.REMAINDER, help="Arguments passed through to the coding agent (prefix with -- )") diff --git a/python/cactus/cli/code.py b/python/cactus/cli/code.py index e563412a9..96adfe2c7 100644 --- a/python/cactus/cli/code.py +++ b/python/cactus/cli/code.py @@ -7,7 +7,7 @@ import urllib.request from pathlib import Path -from .common import BLUE, DEFAULT_MODEL_ID, GREEN, RED, YELLOW, default_backend, is_valid_bundle, print_color, weights_root +from .common import BLUE, DEFAULT_MODEL_ID, GREEN, RED, YELLOW, is_valid_bundle, print_color, weights_root _PACKAGE_DIR = Path(__file__).resolve().parent.parent _PROJECT_ROOT = _PACKAGE_DIR.parent.parent @@ -147,7 +147,7 @@ def cmd_code(args) -> int: "--host", args.host, "--port", str(args.port), "--no-access-log", "--bits", str(getattr(args, "bits", 4)), "--platform", getattr(args, "platform", "auto"), - "--backend", getattr(args, "backend", default_backend()), + "--backend", getattr(args, "backend", "auto"), ] if getattr(args, "token", None): serve_cmd += ["--token", args.token] diff --git a/python/cactus/cli/common.py b/python/cactus/cli/common.py index 370075206..71601e4cb 100644 --- a/python/cactus/cli/common.py +++ b/python/cactus/cli/common.py @@ -1,15 +1,9 @@ #!/usr/bin/env python3 import os -import platform import subprocess import sys from pathlib import Path - -def default_backend() -> str: - """Prefer the Metal GPU backend on Apple hardware, else CPU.""" - return "metal" if platform.system() == "Darwin" else "cpu" - SCRIPT_DIR = Path(__file__).resolve().parent @@ -92,12 +86,6 @@ def apply_runtime_env(args) -> None: and load the stored cloud API key.""" if getattr(args, "no_cloud_tele", False): os.environ["CACTUS_NO_CLOUD_TELE"] = "1" - backend = getattr(args, "backend", None) - if backend == "metal": - os.environ["CACTUS_GPU_FUSED"] = "1" - elif backend == "cpu": - os.environ.pop("CACTUS_GPU_FUSED", None) - os.environ.pop("CACTUS_GPU", None) apply_cloud_api_key_env() From 5ccf7b0baec670135d86fd56248ff16b4482d3c5 Mon Sep 17 00:00:00 2001 From: jakmro Date: Wed, 1 Jul 2026 14:29:15 +0200 Subject: [PATCH 03/36] cleanup v2 Signed-off-by: jakmro --- cactus-engine/src/model.cpp | 42 +++-- cactus-graph/src/execute.cpp | 194 ++-------------------- cactus-graph/src/gpu_fused.cpp | 29 +--- cactus-graph/src/ops_cache.cpp | 3 +- cactus-kernels/src/cactus_kernels.metal | 168 ++++++++++--------- cactus-kernels/src/metal_backend.mm | 76 ++------- cactus-kernels/src/metal_backend_stub.cpp | 2 + 7 files changed, 139 insertions(+), 375 deletions(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 5381569be..3ca261987 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -889,12 +889,15 @@ void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { return; } const bool fused = cactus_backend_fused(); - static const bool nofold = std::getenv("CACTUS_FUSED_NOFOLD") != nullptr; - if (fused && !nofold) { - static int state = 0; static FusedEmbedCtx ctx; - if (state == 0) state = encoder_->graph->extract_ple_pathway(ctx) ? 1 : 2; - if (state == 1) { - ctx.token_id = static_cast(token_id); ctx.position = static_cast(position); + if (fused) { + enum class PleProbe { Untried, Available, Unavailable }; + static PleProbe probe = PleProbe::Untried; + static FusedEmbedCtx ctx; + if (probe == PleProbe::Untried) + probe = encoder_->graph->extract_ple_pathway(ctx) ? PleProbe::Available : PleProbe::Unavailable; + if (probe == PleProbe::Available) { + ctx.token_id = static_cast(token_id); + ctx.position = static_cast(position); cactus_graph_set_fused_embed(&ctx); bool ok = decoder_->graph->execute_gpu_fused(); cactus_graph_set_fused_embed(nullptr); @@ -1915,6 +1918,15 @@ void Model::run_audio_encoder(const std::vector& audio_features) { } } +static float uncertainty_from_margin(float best, float second) { + float confidence = 1.0f; + if (std::isfinite(best) && std::isfinite(second)) { + float margin = std::max(-60.0f, std::min(60.0f, best - second)); + confidence = 1.0f / (1.0f + std::exp(-margin)); + } + return std::max(0.0f, std::min(1.0f, 1.0f - confidence)); +} + uint32_t Model::argmax_logits_at(const BufferDesc& desc, void* ptr, size_t row_off, float* out_uncertainty) { size_t vocab = desc.shape.empty() ? 0 : desc.shape.back(); uint32_t best = 0; @@ -1925,14 +1937,7 @@ uint32_t Model::argmax_logits_at(const BufferDesc& desc, void* ptr, size_t row_o uint32_t gidx; float gbest, gsecond; if (tool_bias.empty() && vocab_bias_.empty() && suppressed_token_id_ < 0 && cactus_graph_gpu_argmax(&gidx, &gbest, &gsecond)) { - if (out_uncertainty) { - float confidence = 1.0f; - if (std::isfinite(gbest) && std::isfinite(gsecond)) { - float margin = std::max(-60.0f, std::min(60.0f, gbest - gsecond)); - confidence = 1.0f / (1.0f + std::exp(-margin)); - } - *out_uncertainty = std::max(0.0f, std::min(1.0f, 1.0f - confidence)); - } + if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(gbest, gsecond); return gidx; } } @@ -1964,14 +1969,7 @@ uint32_t Model::argmax_logits_at(const BufferDesc& desc, void* ptr, size_t row_o int8_t* p = static_cast(ptr) + row_off; for (size_t i = 0; i < vocab; ++i) observe_logit(i, static_cast(p[i])); } - if (out_uncertainty) { - float confidence = 1.0f; - if (std::isfinite(best_v) && std::isfinite(second_v)) { - float margin = std::max(-60.0f, std::min(60.0f, best_v - second_v)); - confidence = 1.0f / (1.0f + std::exp(-margin)); - } - *out_uncertainty = std::max(0.0f, std::min(1.0f, 1.0f - confidence)); - } + if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(best_v, second_v); return best; } diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 031fd3ea0..86244f7e6 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -28,7 +27,7 @@ int cactus_backend_select(const char* backend) { return -1; } bool cactus_backend_metal() { return g_backend_override != 0 && cactus_metal_available(); } -bool cactus_backend_fused() { return g_backend_override != 0 && cactus_metal_available(); } +bool cactus_backend_fused() { return cactus_backend_metal(); } using ComputeFn = void(*)(GraphNode&, const nodes_vector&, const node_index_map_t&); @@ -447,12 +446,6 @@ void CactusGraph::infer_shapes() { runtime_shapes_dirty_ = false; } -static bool gpu_op_enabled(const char* name) { - static const std::string spec = []{ const char* e=std::getenv("CACTUS_GPU_OPS"); return e?std::string(e):std::string(); }(); - if (spec.empty()) return true; - return spec.find(name) != std::string::npos; -} - static void row_strides(const std::vector& shape, size_t* out) { size_t s=1; for (int k=(int)shape.size()-1; k>=0; --k){ out[k]=s; s*=shape[k]; } } @@ -466,29 +459,6 @@ static void bcast_strides(const std::vector& in_shape, const std::vector static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const node_index_map_t& map) { BufferDesc& out = node.output_buffer; auto fp16 = [](const BufferDesc& b){ return b.precision == Precision::FP16; }; - if (std::getenv("CACTUS_GPU_FALLBACK")) return false; - { - const char* g = nullptr; - switch (node.op_type) { - case OpType::VIEW: case OpType::RESHAPE: case OpType::FLATTEN: g="copy"; break; - case OpType::MATMUL: g="matmul"; break; - case OpType::ADD: case OpType::ADD_CLIPPED: case OpType::SUBTRACT: - case OpType::MULTIPLY: case OpType::DIVIDE: g="binary"; break; - case OpType::SCALAR_ADD: case OpType::SCALAR_SUBTRACT: - case OpType::SCALAR_MULTIPLY: case OpType::SCALAR_DIVIDE: g="scalar"; break; - case OpType::GELU: case OpType::TANH: case OpType::SILU: case OpType::RELU: g="unary"; break; - case OpType::RMS_NORM: g="rms"; break; - case OpType::PRECISION_CAST: g="cast"; break; - case OpType::ATTENTION_CACHED: g="attn"; break; - case OpType::TRANSPOSE: g="transpose"; break; - case OpType::SLICE: g="slice"; break; - case OpType::INDEX: g="index"; break; - case OpType::CAT: g="cat"; break; - case OpType::KV_CACHE_APPEND: g="kvappend"; break; - default: g=nullptr; - } - if (g && !gpu_op_enabled(g)) return false; - } switch (node.op_type) { case OpType::VIEW: case OpType::RESHAPE: case OpType::FLATTEN: { const auto& in = get_input(node, 0, nodes, map); @@ -614,12 +584,6 @@ static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const nod const auto& vnew = get_input(node, 2, nodes, map); const auto& kcache = get_input(node, 3, nodes, map); const auto& vcache = get_input(node, 4, nodes, map); - if (std::getenv("CACTUS_GPU_DEBUG")) { - static int dq=0; - if (qb.shape.size()<2 || qb.shape[1]==1) if (dq++ < 4) std::cerr << "[attn] q.shape.size=" << qb.shape.size() - << " dims=[" << (qb.shape.size()>0?qb.shape[0]:0) << "," << (qb.shape.size()>1?qb.shape[1]:0) - << "," << (qb.shape.size()>2?qb.shape[2]:0) << "] kc.prec=" << (int)kcache.precision << "\n"; - } if (qb.shape.size() < 3) return false; size_t batch=qb.shape[0], seq=qb.shape[1], nqh=qb.shape[2]; if (batch != 1) return false; @@ -668,10 +632,6 @@ static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const nod (uint32_t)history_len, (uint32_t)total_keys, (uint32_t)kv_start, (uint32_t)kv_end, scale, history_len*kv_heads*hdim, history_len*kv_heads*v_hdim, history_len*kv_heads*ngK*sizeof(float), history_len*kv_heads*ngV*sizeof(float)); - if (std::getenv("CACTUS_GPU_DEBUG")) { static int d2=0; if (d2++ < 3) - std::cerr << "[attn2] nqh="< ok="< 0 && ws < ceiling; + size_t new_total = current_len + new_seq_len; + char* base = static_cast(cache.get_data()); + size_t ngK = (hdim + 31)/32; if (new_seq_len > 1) { - size_t ceiling = cache_node.params.max_cache_seq_len, ws = node.params.window_size; - bool sliding = ws > 0 && ws < ceiling; - size_t new_total = current_len + new_seq_len; - char* base = static_cast(cache.get_data()); - size_t ngK = (hdim + 31)/32; if (sliding) { uint32_t W = (uint32_t)(max_len - sink - 1); if (!cactus_metal_encode_kv_append_ring_i8_m(new_kv.get_data(), base + 64, @@ -728,11 +688,6 @@ static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const nod if (out.get_data()) *out.data_as() = static_cast(km[0]); return true; } - size_t ceiling = cache_node.params.max_cache_seq_len, ws = node.params.window_size; - bool sliding = ws > 0 && ws < ceiling; - size_t new_total = current_len + new_seq_len; - char* base = static_cast(cache.get_data()); - size_t ngK = (hdim + 31)/32; if (!sliding && new_total > max_len) { if (!cactus_kv_cache_grow(cache, new_total, ceiling)) return false; km = reinterpret_cast(cache.get_data()); @@ -792,80 +747,10 @@ static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const nod } } -namespace { -struct CpuOpProf { double ns = 0.0; uint64_t inv = 0; }; -std::map> g_cpu_prof; -std::map g_cpu_prof_calls; -void cpu_prof_dump() { - for (auto& nc : g_cpu_prof) { - size_t nn = nc.first; uint64_t calls = g_cpu_prof_calls[nn]; - if (!calls) continue; - double tot = 0.0; for (auto& op : nc.second) tot += op.second.ns; - std::vector> v(nc.second.begin(), nc.second.end()); - std::sort(v.begin(), v.end(), [](const std::pair& a, const std::pair& b){ - return a.second.ns > b.second.ns; }); - std::cerr << "\n=== CPU op profile: graph n=" << nn << " | " << calls << " call(s) | " - << std::fixed << std::setprecision(3) << (tot/calls/1e6) << " ms/call total ===\n"; - std::cerr << " " << std::left << std::setw(24) << "op" << std::right - << std::setw(11) << "ms/call" << std::setw(11) << "cnt/call" << std::setw(9) << "share\n"; - for (auto& e : v) { - std::cerr << " " << std::left << std::setw(24) << get_op_name((OpType)e.first) << std::right - << std::fixed << std::setprecision(4) << std::setw(11) << (e.second.ns/calls/1e6) - << std::setprecision(1) << std::setw(11) << ((double)e.second.inv/calls) - << std::setw(8) << (100.0*e.second.ns/tot) << "%\n"; - } - } -} -inline void cpu_prof_begin(size_t n) { - static bool reg = false; - if (!reg) { std::atexit(cpu_prof_dump); reg = true; } - g_cpu_prof_calls[n]++; -} -} - void CactusGraph::execute(const std::string& profile_file) { BufferPool& pool = buffer_pool_; const size_t n = nodes_.size(); - static const bool gpu_timing = std::getenv("CACTUS_GPU_TIMING") != nullptr; - auto _tt = [](){ return std::chrono::high_resolution_clock::now(); }; - auto _t0 = _tt(); infer_shapes(); - auto _t_infer = _tt(); - - if (std::getenv("CACTUS_OP_HIST")) { - static std::unordered_set seen_sizes; - if (seen_sizes.insert(n).second) { - std::unordered_map hist; - for (size_t i = 0; i < n; ++i) hist[static_cast(nodes_[i]->op_type)]++; - std::cerr << "[ophist] graph nodes=" << n << "\n"; - for (auto& kv : hist) - std::cerr << " " << get_op_name(static_cast(kv.first)) - << " : " << kv.second << "\n"; - } - } - - if (std::getenv("CACTUS_GRAPH_DUMP")) { - static std::unordered_set dumped; - if (dumped.insert(n).second) { - auto shp=[](const BufferDesc&b){ std::string s="["; for(size_t d=0;dsecond:SIZE_MAX); if(j)std::cerr<<","; std::cerr<<(long)idx; } - std::cerr << "]"; - if (nd.params.axis) std::cerr << " axis=" << nd.params.axis; - if (nd.params.epsilon != 0.0f) std::cerr << " eps=" << nd.params.epsilon; - if (nd.params.scalar != 0.0f) std::cerr << " scalar=" << nd.params.scalar; - if (!nd.params.permutation.empty()){ std::cerr<<" perm=["; for(size_t d=0;d int { const char* val = std::getenv(name); @@ -936,7 +821,6 @@ void CactusGraph::execute(const std::string& profile_file) { }; const bool gpu_mode = cactus_backend_metal(); - const bool gpu_mode_req = gpu_mode && !g_cactus_force_cpu; auto aliases_input = [&](const GraphNode& node) -> bool { if (node.op_type == OpType::SLICE && !node.input_ids.empty()) { auto it = node_index_map_.find(node.input_ids[0]); @@ -951,7 +835,7 @@ void CactusGraph::execute(const std::string& profile_file) { } } if (node.op_type == OpType::INDEX && node.params.axis == 0) return true; - if (!gpu_mode_req) return false; + if (!gpu_mode) return false; if (node.op_type == OpType::VIEW || node.op_type == OpType::RESHAPE || node.op_type == OpType::FLATTEN) return true; if (node.op_type == OpType::PRECISION_CAST && !node.input_ids.empty()) { @@ -996,12 +880,9 @@ void CactusGraph::execute(const std::string& profile_file) { release_after[last_use[i]].push_back(i); } - static const bool gpu_verify = std::getenv("CACTUS_GPU_VERIFY") != nullptr; - if (gpu_mode_req && !need_debug) { - auto _t_live = _tt(); + if (gpu_mode && !need_debug) { cactus_metal_session_begin(); cactus_metal_set_active(true); - std::unordered_map gpu_fallback_hist; auto gpu_release = [&](size_t idx) { GraphNode& nd = *nodes_[idx]; if (aliases_input(nd)) nd.output_buffer.external_data = nullptr; @@ -1065,69 +946,20 @@ void CactusGraph::execute(const std::string& profile_file) { hazard_barrier(*node); bool encoded = try_encode_gpu(*node, nodes_, node_index_map_); if (!encoded) { - gpu_fallback_hist[(int)ot]++; cactus_metal_session_sync(); accessed.clear(); dispatch_node(*node, nodes_, node_index_map_); - } else if (gpu_verify) { - cactus_metal_session_sync(); - BufferDesc& ob = node->output_buffer; - std::vector gpu_copy(ob.byte_size); - std::memcpy(gpu_copy.data(), ob.get_data(), ob.byte_size); - dispatch_node(*node, nodes_, node_index_map_); - if (ob.precision == Precision::FP16) { - const __fp16* g = reinterpret_cast(gpu_copy.data()); - const __fp16* c = ob.data_as<__fp16>(); - double maxd=0; size_t bad=0; - for (size_t k=0;kmaxd)maxd=d; if(d>0.05)bad++; } - if (maxd > 0.05) { - static std::set seen; - std::string key = std::string(get_op_name(node->op_type)); - if (seen.insert(key).second) - std::cerr << "[gpu-verify] MISMATCH op=" << get_op_name(node->op_type) - << " id=" << node->id << " maxdiff=" << maxd << " bad=" << bad - << "/" << ob.total_size << " shape0=" << (ob.shape.empty()?0:ob.shape[0]) << "\n"; - } - } } if (ot == OpType::PERSISTENT) populated_node_ids_.insert(node->id); for (size_t r : release_after[i]) gpu_release(r); } - if (std::getenv("CACTUS_GPU_STATS") && n > 1000) { - static std::set seen; - if (seen.insert(n).second) { std::cerr << "[gpu-fallback] graph n=" << n << ":\n"; - for (auto& kv : gpu_fallback_hist) std::cerr << " " << get_op_name((OpType)kv.first) << " : " << kv.second << "\n"; } - } cactus_metal_set_active(false); - auto _t_loop = _tt(); cactus_metal_session_end(); - if (gpu_timing && n > 1000) { - using us = std::chrono::microseconds; - auto _t_done = _tt(); - static int _c = 0; static long _sum_total=0, _sum_loop=0; static int _cnt=0; - _sum_total += std::chrono::duration_cast(_t_done-_t0).count(); - _sum_loop += std::chrono::duration_cast(_t_loop-_t_live).count(); - _cnt++; - if (_c++ % 32 == 31) - std::cerr << "[gpu-timing] avg total execute=" << _sum_total/_cnt - << "us loop(encode+inner-sync)=" << _sum_loop/_cnt - << "us (CPU encode = total - on-GPU)\n"; - } return; } if (!need_debug) { - static const bool cpu_prof = std::getenv("CACTUS_CPU_PROFILE") != nullptr; - if (cpu_prof) cpu_prof_begin(n); - auto run = [&](auto& nd) { - if (!cpu_prof) { dispatch_node(nd, nodes_, node_index_map_); return; } - auto t0 = std::chrono::high_resolution_clock::now(); - dispatch_node(nd, nodes_, node_index_map_); - auto& pr = g_cpu_prof[n][(int)nd.op_type]; - pr.ns += (double)std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - t0).count(); - pr.inv++; - }; + auto run = [&](auto& nd) { dispatch_node(nd, nodes_, node_index_map_); }; for (size_t i = 0; i < n; ++i) { auto& node = nodes_[i]; if (node->op_type == OpType::INPUT) continue; @@ -1146,12 +978,6 @@ void CactusGraph::execute(const std::string& profile_file) { } run(*node); trace_nonfinite(i, *node); - if (n == 3861) { static const char* dn = std::getenv("CACTUS_DUMP_NODE"); - if (dn && i == (size_t)std::atoi(dn)) { static int dc=0; if (dc++<1) { - BufferDesc& b = node->output_buffer; const __fp16* p = b.data_as<__fp16>(); - if (p) std::cerr << "[dump] node " << i << " " << get_op_name(node->op_type) << ": " - << (float)p[0] << " " << (float)p[1] << " " << (float)p[2] << " " << (float)p[3] - << " | " << (float)p[4] << " " << (float)p[5] << "\n"; } } } if (node->op_type == OpType::PERSISTENT) { populated_node_ids_.insert(node->id); } @@ -1625,7 +1451,7 @@ void CactusGraph::soft_reset_keep_pool() { } void CactusGraph::prewarm_gpu_quant_weights() { - const bool gpu = cactus_backend_metal() && std::getenv("CACTUS_NO_PREWARM") == nullptr; + const bool gpu = cactus_backend_metal(); if (!gpu) return; for (auto& np : nodes_) { GraphNode& node = *np; diff --git a/cactus-graph/src/gpu_fused.cpp b/cactus-graph/src/gpu_fused.cpp index cb3a06bc6..1cb970e42 100644 --- a/cactus-graph/src/gpu_fused.cpp +++ b/cactus-graph/src/gpu_fused.cpp @@ -98,7 +98,6 @@ bool CactusGraph::extract_ple_pathway(FusedEmbedCtx& ctx) const { } bool CactusGraph::execute_gpu_fused() { - static const bool dbg = std::getenv("CACTUS_FUSED_DEBUG") != nullptr; if (!cactus_metal_available()) return false; const size_t n = nodes_.size(); if (n < 3861) return false; @@ -164,18 +163,12 @@ bool CactusGraph::execute_gpu_fused() { for (int l = 0; l < 15; ++l) { const uint64_t* km = (const uint64_t*)G.L[l].kc; - if (G.L[l].global && km[0] + 1 > km[1]) { - static const bool dbg2 = (std::getenv("CACTUS_FUSED_TIMER") != nullptr); - if (dbg2) { static int n=0; if (n++<3) fprintf(stderr,"[fused-bail] L%d clen=%llu max=%llu hd=%d\n", - l,(unsigned long long)km[0],(unsigned long long)km[1],G.L[l].hd); } - return false; - } + if (G.L[l].global && km[0] + 1 > km[1]) return false; } G.arena.reset(); auto fresh = [&](size_t elems){ return G.arena.fresh(elems * 2); }; cactus_metal_session_begin(); - auto g_t0 = std::chrono::high_resolution_clock::now(); cactus_metal_set_active(true); void* h = fresh(HID); @@ -202,10 +195,6 @@ bool CactusGraph::execute_gpu_fused() { std::memcpy(h, G.hidden, HID*2); pleBase = G.ple; } - if (dbg) { cactus_metal_session_sync(); const __fp16* hh=(const __fp16*)h; const __fp16* pp=(const __fp16*)pleBase; - fprintf(stderr,"[embed %s] tok=%d h[0..3]= %.5f %.5f %.5f %.5f | ple[0..3]= %.5f %.5f %.5f %.5f\n", - fold?"FOLD":"node", fold?g_fe.token_id:-1, (float)hh[0],(float)hh[1],(float)hh[2],(float)hh[3], - (float)pp[0],(float)pp[1],(float)pp[2],(float)pp[3]); } auto gather = [&](const void* tbl, void* dst, int hd){ std::memcpy(dst, (const char*)tbl + (size_t)pos*hd*2, hd*2); }; gather(G.cosL, G.cos_s, 256); gather(G.sinL, G.sin_s, 256); @@ -280,9 +269,6 @@ bool CactusGraph::execute_gpu_fused() { float ls = (float)(*(const __fp16*)w.scalar); void* h4 = fresh(HID); cactus_metal_encode_scalar(2, h4, h, HID, ls); h = h4; - - if (dbg) { cactus_metal_session_sync(); const __fp16* p=(const __fp16*)h; - fprintf(stderr,"[fused] L%d out: %.5f %.5f %.5f %.5f (ls=%.6f)\n",l,(float)p[0],(float)p[1],(float)p[2],(float)p[3],ls); } } size_t V = B(3860).total_size; @@ -306,17 +292,6 @@ bool CactusGraph::execute_gpu_fused() { } cactus_metal_set_active(false); - static const bool g_tmr = (std::getenv("CACTUS_FUSED_TIMER") != nullptr); - if (g_tmr) { - auto t1 = std::chrono::high_resolution_clock::now(); - cactus_metal_session_end(); - auto t2 = std::chrono::high_resolution_clock::now(); - double enc = std::chrono::duration(t1 - g_t0).count(); - double syn = std::chrono::duration(t2 - t1).count(); - fprintf(stderr, "[fused-timer] pos=%d encode_cpu=%.2fms commit+wait=%.2fms total=%.2fms\n", - pos, enc, syn, enc+syn); - } else { - cactus_metal_session_end(); - } + cactus_metal_session_end(); return true; } diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 6fbadf68a..9120e86b0 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -101,6 +101,7 @@ inline bool use_fp16_kv_cache() { } constexpr size_t kInitialCacheEntries = 256; +constexpr size_t kMetalInitialCacheEntries = 4000; inline bool kv_cache_resident() { return cactus_backend_metal(); @@ -179,7 +180,7 @@ void compute_kv_cache_state_node( size_t max_seq; if (sliding) max_seq = std::min(ceiling, window + node.params.cache_sink_size + 1); else if (num_slots > 1) max_seq = ceiling; - else if (kv_cache_resident()) max_seq = std::min(ceiling, static_cast(4000)); + else if (kv_cache_resident()) max_seq = std::min(ceiling, kMetalInitialCacheEntries); else max_seq = std::min(ceiling, kInitialCacheEntries); size_t kv_heads = node.params.num_kv_heads; size_t hdim = node.params.head_dim; diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 73f97cc4c..5234a7025 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -439,11 +439,16 @@ kernel void scalar_f16(device const half* in [[buffer(0)]], device half* y [[buf y[i]=(half)r; } +inline float gelu_tanh(float x) { + float c = 0.7978845608028654f * (x + 0.044715f*x*x*x); + return 0.5f * x * (1.0f + precise::tanh(c)); +} + kernel void unary_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], uint i [[thread_position_in_grid]]) { if (i>=n) return; float x=(float)in[i], r; - if (op==0){ float c=0.7978845608028654f*(x+0.044715f*x*x*x); r=0.5f*x*(1.0f+precise::tanh(c)); } + if (op==0) r=gelu_tanh(x); else if (op==1) r=precise::tanh(x); else if (op==2) r=x/(1.0f+precise::exp(-x)); else r=max(x,0.0f); @@ -455,8 +460,7 @@ kernel void swiglu_f16(device const half* gate [[buffer(0)]], device const half* constant float& scale [[buffer(4)]], uint i [[thread_position_in_grid]]) { if (i>=n) return; float x=(float)gate[i]; - float c=0.7978845608028654f*(x+0.044715f*x*x*x); - half g1=(half)(0.5f*x*(1.0f+precise::tanh(c))); + half g1=(half)gelu_tanh(x); half g2=(half)((float)g1*scale); y[i]=(half)((float)g2*(float)up[i]); } @@ -822,108 +826,108 @@ kernel void attn_prefill_i8( return; } if (ringR > 0u) { - float lmax = -INFINITY; - for (uint j = t; j < nactive; j += T) { - uint k = (j < S) ? j : rstart + (j - S); - float dot = 0; - if (k < history_len) { - uint slot = (ringR > 0u && k >= Wsr) ? (S + ((k - S) % ringR)) : k; - device const char* kk = kc + ((size_t)slot*num_kv_heads + kvh)*head_dim; - device const float* kss = ks + ((size_t)slot*num_kv_heads + kvh)*ngK; - for (uint d=0; d0;s>>=1){ if(t0;s>>=1){ if(t 0 ? 1.0f/red[0] : 0.0f; - threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint d = t; d < v_hdim; d += T) { - float acc=0; - for (uint j=0; j 0u && k >= Wsr) ? (S + ((k - S) % ringR)) : k; - device const char* vvv = vc + ((size_t)slot*num_kv_heads+kvh)*v_hdim; - device const float* vss = vs + ((size_t)slot*num_kv_heads+kvh)*ngV; - vv = (float)vvv[d]*vss[d/32]; - } else { - device const half* vvv = vnew + ((size_t)(k-history_len)*num_kv_heads+kvh)*v_hdim; - vv = (float)vvv[d]; - } - acc += sc[j]*vv; - } - out[((size_t)m*num_q_heads + h)*v_hdim + d] = (half)(acc*inv); - } - } else { - uint TILE = maxsc; - float rmax = -INFINITY, rsum = 0.0f; - float acc[8]; - for (uint a=0;a<8;++a) acc[a]=0.0f; - for (uint tile=0; tile 0u && k >= Wsr) ? (S + ((k - S) % ringR)) : k; + device const char* kk = kc + ((size_t)slot*num_kv_heads + kvh)*head_dim; + device const float* kss = ks + ((size_t)slot*num_kv_heads + kvh)*ngK; for (uint d=0; d0;s>>=1){ if(t0;s>>=1){ if(t 0 ? 1.0f/red[0] : 0.0f; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint d = t; d < v_hdim; d += T) { + float acc=0; + for (uint j=0; j= Wsr) ? (S + ((k - S) % ringR)) : k; + device const char* vvv = vc + ((size_t)slot*num_kv_heads+kvh)*v_hdim; + device const float* vss = vs + ((size_t)slot*num_kv_heads+kvh)*ngV; vv = (float)vvv[d]*vss[d/32]; } else { device const half* vvv = vnew + ((size_t)(k-history_len)*num_kv_heads+kvh)*v_hdim; vv = (float)vvv[d]; } - a += sc[jj]*vv; + acc += sc[j]*vv; } - acc[ai]=a; ++ai; + out[((size_t)m*num_q_heads + h)*v_hdim + d] = (half)(acc*inv); } - rmax = rmax_new; - threadgroup_barrier(mem_flags::mem_threadgroup); - } - float inv = rsum > 0 ? 1.0f/rsum : 0.0f; - uint ai=0; - for (uint d=t; d0;s>>=1){ if(t0;s>>=1){ if(t 0 ? 1.0f/rsum : 0.0f; + uint ai=0; + for (uint d=t; dsecond, static_cast(a - base) }; } - static const bool g_replay = (std::getenv("CACTUS_GPU_REPLAY") != nullptr); size_t nb = bytes ? bytes : 1; - if (g_replay) { - static std::unordered_map> g_snap; - auto sit = g_snap.find(p); - id b; - if (sit != g_snap.end() && (size_t)sit->second.length >= nb) b = sit->second; - else { b = [ctx().dev newBufferWithLength:nb options:MTLResourceStorageModeShared]; g_snap[p] = b; } - std::memcpy([b contents], p, nb); - return { b, 0 }; - } id b = recycled(bytes); std::memcpy([b contents], p, nb); return { b, 0 }; @@ -213,12 +203,6 @@ inline void setBufAt(const void* p, size_t bytes, int idx) { [g_enc setBuffer:pr.first offset:pr.second atIndex:idx]; } -inline void setBufFresh(const void* p, size_t bytes, int idx) { - id b = recycled(bytes); - std::memcpy([b contents], p, bytes ? bytes : 1); - [g_enc setBuffer:b offset:0 atIndex:idx]; -} - } bool cactus_metal_available() { return ctx().ok; } @@ -229,24 +213,17 @@ inline void setBufFresh(const void* p, size_t bytes, int idx) { bool cactus_metal_concurrent() { return g_concurrent; } void cactus_metal_barrier() { if (g_concurrent && g_enc) [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; } -void cactus_metal_session_begin() { } -static int g_sync_count = 0; -static double g_gpu_ms = 0; +void cactus_metal_session_begin() {} void cactus_metal_session_sync() { if (g_enc) { [g_enc endEncoding]; [g_cmd commit]; [g_cmd waitUntilCompleted]; - g_gpu_ms += ([g_cmd GPUEndTime] - [g_cmd GPUStartTime]) * 1000.0; - g_enc = nil; g_cmd = nil; ++g_sync_count; + g_enc = nil; g_cmd = nil; } for (id b : g_pending) g_free[(size_t)b.length].push_back(b); g_pending.clear(); } void cactus_metal_session_end() { cactus_metal_session_sync(); - if (getenv("CACTUS_GPU_STATS")) { - fprintf(stderr, "[gpu-stats] syncs=%d on-GPU=%.3f ms\n", g_sync_count, g_gpu_ms); - g_sync_count = 0; g_gpu_ms = 0; - } } void* cactus_metal_alloc_shared(size_t bytes) { @@ -395,6 +372,12 @@ bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_pr barrier(); return true; } +static inline bool quant_fast_eligible(const CactusQuantMatrix* W) { + return W->bits == 4 && W->group_size >= 128 && (W->group_size % 128) == 0 && (W->N % 4) == 0 && + !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->input_scale_recip && W->left_signs && + W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; +} + bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W) { const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; if (ctx().ok && (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->rotation && ng==1 && gs==K && (N%4)==0) { @@ -402,9 +385,7 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu if (ortho_code_k < K) { ortho_code = cactus_metal_alloc_shared((size_t)K*sizeof(__fp16)); ortho_code_k = K; } if (ortho_code) return cactus_metal_encode_quant_matmul_ortho(out, lhs, ortho_code, W); } - bool fast = ctx().ok && W->bits==4 && gs>=128 && (gs%128)==0 && (N%4)==0 && - !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->input_scale_recip && W->left_signs && - W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; + bool fast = ctx().ok && quant_fast_eligible(W); if (!fast) return false; ResW& rw = resident(W); @@ -434,11 +415,7 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { if (!ctx().ok) return false; - const uint32_t gs=W->group_size, N=W->N; - bool fast = W->bits==4 && gs>=128 && (gs%128)==0 && (N%4)==0 && - !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->input_scale_recip && W->left_signs && - W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; - if (!fast) return false; + if (!quant_fast_eligible(W)) return false; resident(W); return true; } @@ -446,9 +423,7 @@ bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M) { if (M == 1) return cactus_metal_encode_quant_matmul(out, lhs, W); const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; - bool fast = ctx().ok && W->bits==4 && gs>=128 && (gs%128)==0 && (N%4)==0 && - !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->input_scale_recip && W->left_signs && - W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; + bool fast = ctx().ok && quant_fast_eligible(W); if (!fast) return false; ResW& rw = resident(W); size_t code_bytes = (size_t)M*ng*gs*sizeof(__fp16); @@ -673,13 +648,10 @@ bool cactus_metal_encode_attention_i8( setCache(ks, ks_bytes, 5); setCache(vs, vs_bytes, 6); }; - static const bool g_nosplit = (std::getenv("CACTUS_ATTN_NOSPLIT") != nullptr); - static const uint32_t CHUNK_BASE = std::getenv("CACTUS_ATTN_CHUNK") ? (uint32_t)atoi(std::getenv("CACTUS_ATTN_CHUNK")) : 96; - static const uint32_t C_MAX = std::getenv("CACTUS_ATTN_CMAX") ? (uint32_t)atoi(std::getenv("CACTUS_ATTN_CMAX")) : 64; + constexpr uint32_t kAttnChunkKeys = 96, kAttnMaxParts = 64; const uint32_t R = kv_end - kv_start; - uint32_t chunk = CHUNK_BASE, C = (R + chunk - 1) / chunk; - if (C > C_MAX) { C = C_MAX; chunk = (R + C - 1) / C; } - if (g_nosplit) { C = 1; chunk = R; } + uint32_t chunk = kAttnChunkKeys, C = (R + chunk - 1) / chunk; + if (C > kAttnMaxParts) { C = kAttnMaxParts; chunk = (R + C - 1) / C; } if (C <= 1) { const uint32_t T = 256; @@ -698,7 +670,7 @@ bool cactus_metal_encode_attention_i8( uint32_t TP = chunk < 128 ? chunk : 128; TP = (TP + 31u) & ~31u; - if (TP < 32) TP = 32; if (TP > 128) TP = 128; + if (TP < 32) TP = 32; id partO = recycled((size_t)num_q_heads*C*v_hdim*sizeof(float)); id partM = recycled((size_t)num_q_heads*C*sizeof(float)); id partL = recycled((size_t)num_q_heads*C*sizeof(float)); @@ -946,21 +918,7 @@ bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, uint32_t M, __fp16* C) { const uint32_t gs = W->group_size; - const bool fast = - ctx().ok && M == 1 && W->bits == 4 && gs >= 128 && (gs % 128) == 0 && - !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && (W->N % 4) == 0 && - W->input_scale_recip != nullptr && W->left_signs != nullptr && - W->right_signs != nullptr && W->permutation != nullptr && - W->codebook != nullptr && W->norms != nullptr && W->packed_indices != nullptr; - - if (getenv("CACTUS_GPU_DEBUG") && M == 1) { - static std::atomic dbg{0}; - if (dbg++ < 14) - fprintf(stderr, "[metal] M=%u bits=%u gs=%u flags=0x%x recip=%d ls=%d rs=%d perm=%d -> %s\n", - M, W->bits, gs, W->flags, W->input_scale_recip != nullptr, - W->left_signs != nullptr, W->right_signs != nullptr, - W->permutation != nullptr, fast ? "GPU" : "CPU-fallback"); - } + const bool fast = ctx().ok && M == 1 && quant_fast_eligible(W); if (!fast) { cactus_quant_matmul(W, A, M, C); return; } diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 778f4a976..8e54eb5d1 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -51,6 +51,8 @@ bool cactus_metal_encode_kv_append_i8(const void*, void*, void*, uint32_t, uint3 uint32_t, size_t, size_t, size_t) { return false; } bool cactus_metal_encode_kv_append_sliding_i8(const void*, void*, void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_kv_append_sliding_i8_m(const void*, void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } bool cactus_metal_encode_kv_append_i8_m(const void*, void*, void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } bool cactus_metal_encode_kv_append_ring_i8_m(const void*, void*, void*, uint32_t, uint32_t, uint32_t, From e81cc3948506ff9641cd3c3e139c33a9aa2614df Mon Sep 17 00:00:00 2001 From: jakmro Date: Wed, 1 Jul 2026 16:30:41 +0200 Subject: [PATCH 04/36] faster Signed-off-by: jakmro --- cactus-engine/test.sh | 8 +- cactus-engine/tests/test_needle.cpp | 1 + cactus-engine/tests/test_utils.cpp | 15 ++ cactus-engine/tests/test_utils.h | 3 + cactus-graph/src/ops_cache.cpp | 3 - cactus-graph/src/ops_math.cpp | 3 - cactus-kernels/src/cactus_kernels.metal | 199 ++++++++---------------- cactus-kernels/src/metal_backend.mm | 82 ++++------ python/cactus/cli/__init__.py | 2 + python/cactus/cli/test.py | 1 + 10 files changed, 124 insertions(+), 193 deletions(-) diff --git a/cactus-engine/test.sh b/cactus-engine/test.sh index 25cfc92ec..7d581662c 100755 --- a/cactus-engine/test.sh +++ b/cactus-engine/test.sh @@ -8,6 +8,7 @@ ASSETS_DIR="$(pwd)/tests/assets" IOS_MODE=false ANDROID_MODE=false SUITE="" +BACKEND="auto" while [[ $# -gt 0 ]]; do case $1 in @@ -16,6 +17,7 @@ while [[ $# -gt 0 ]]; do --suite) SUITE="${2:?--suite needs an argument}"; shift 2 ;; --model) CACTUS_TEST_MODEL="${2:?--model needs an argument}"; shift 2 ;; --transcription-model) CACTUS_TEST_TRANSCRIPTION_MODEL="${2:?--transcription-model needs an argument}"; shift 2 ;; + --backend) BACKEND="${2:?--backend needs an argument}"; shift 2 ;; *) echo "Unknown arg: $1" >&2; exit 2 ;; esac done @@ -33,11 +35,11 @@ BUNDLE_DIR="$(require_bundle "$CACTUS_TEST_MODEL" "model")" TRANSCRIPTION_BUNDLE_DIR="$(require_bundle "$CACTUS_TEST_TRANSCRIPTION_MODEL" "transcription-model")" if [ "$IOS_MODE" = true ]; then - export CACTUS_TEST_MODEL="$BUNDLE_DIR" CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" CACTUS_TEST_SUITE="$SUITE" + export CACTUS_TEST_MODEL="$BUNDLE_DIR" CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" CACTUS_TEST_SUITE="$SUITE" CACTUS_TEST_BACKEND="$BACKEND" exec "$(pwd)/tests/ios/run.sh" fi if [ "$ANDROID_MODE" = true ]; then - export CACTUS_TEST_MODEL="$BUNDLE_DIR" CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" CACTUS_TEST_SUITE="$SUITE" + export CACTUS_TEST_MODEL="$BUNDLE_DIR" CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" CACTUS_TEST_SUITE="$SUITE" CACTUS_TEST_BACKEND="$BACKEND" exec "$(pwd)/tests/android/run.sh" fi @@ -45,6 +47,7 @@ echo "Model: $CACTUS_TEST_MODEL" echo "Bundle: $BUNDLE_DIR" echo "Transcription model: $CACTUS_TEST_TRANSCRIPTION_MODEL" echo "Transcription bundle: $TRANSCRIPTION_BUNDLE_DIR" +echo "Backend: $BACKEND" cd "$PROJECT_ROOT/cactus-engine/tests" rm -rf build && mkdir build && cd build @@ -55,6 +58,7 @@ export CACTUS_TEST_MODEL="$BUNDLE_DIR" export CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" export CACTUS_TEST_ASSETS="$ASSETS_DIR" export CACTUS_INDEX_PATH="$ASSETS_DIR" +export CACTUS_TEST_BACKEND="$BACKEND" FAILED=0 if [ -n "$SUITE" ]; then diff --git a/cactus-engine/tests/test_needle.cpp b/cactus-engine/tests/test_needle.cpp index 905672a09..4f114e8b3 100644 --- a/cactus-engine/tests/test_needle.cpp +++ b/cactus-engine/tests/test_needle.cpp @@ -90,6 +90,7 @@ bool test_needle_tool_call() { } int main() { + TestUtils::apply_backend(); bool ok = true; ok &= test_needle_tool_call(); std::cout << "\n" << (ok ? "✓ needle tool-call test passed" : "✗ needle tool-call test failed") << "\n"; diff --git a/cactus-engine/tests/test_utils.cpp b/cactus-engine/tests/test_utils.cpp index 955095622..1dd50fcf4 100644 --- a/cactus-engine/tests/test_utils.cpp +++ b/cactus-engine/tests/test_utils.cpp @@ -1,10 +1,25 @@ #include "test_utils.h" #include +#include +#include namespace TestUtils { +void apply_backend() { + static bool applied = false; + if (applied) return; + applied = true; + const char* b = std::getenv("CACTUS_TEST_BACKEND"); + if (!b || !*b || std::strcmp(b, "auto") == 0) return; + if (cactus_set_backend(b) == 0) + std::cout << "Backend: " << (std::strcmp(b, "metal") == 0 ? "Metal GPU" : "CPU") << "\n"; + else + std::cout << "Backend '" << b << "' unavailable; using default\n"; +} + TestRunner::TestRunner(const std::string& suite_name) : suite_name_(suite_name), passed_count_(0), total_count_(0) { + apply_backend(); std::cout << "\n╔══════════════════════════════════════════════════════════════════════════════════════╗\n" << "║ Running " << std::left << std::setw(76) << suite_name_ << " ║\n" << "╚══════════════════════════════════════════════════════════════════════════════════════╝\n"; diff --git a/cactus-engine/tests/test_utils.h b/cactus-engine/tests/test_utils.h index 19aba3819..ab7c96c9f 100644 --- a/cactus-engine/tests/test_utils.h +++ b/cactus-engine/tests/test_utils.h @@ -27,6 +27,9 @@ class TestRunner { int total_count_; }; +// Select the inference backend from CACTUS_TEST_BACKEND ("auto"/"cpu"/"metal"). Applied once. +void apply_backend(); + } namespace EngineTestUtils { diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 9120e86b0..5681a3728 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -120,9 +120,6 @@ inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { size_t total = fp16_cache ? fp16_cache_elements(new_max, kv_heads, hdim) : cache_buffer_size(new_max, kv_heads, hdim); BufferDesc resized({total}, fp16_cache ? Precision::FP16 : Precision::INT8); - // Keep the cache GPU-resident across grows: a plain CPU realloc would drop it out of the Metal - // shared map, forcing the fused/executor decode to snapshot the whole KV every token (O(context) - // CPU cost). Allocating shared keeps the live, offset-aware read path. CPU build: active=false. bool shared = kv_cache_resident(); void* old_data = buf.get_data(); if (shared) { diff --git a/cactus-graph/src/ops_math.cpp b/cactus-graph/src/ops_math.cpp index 771bee1dd..e12661268 100644 --- a/cactus-graph/src/ops_math.cpp +++ b/cactus-graph/src/ops_math.cpp @@ -595,9 +595,6 @@ void compute_reshape_node(GraphNode& node, const std::vector(input_buffer.get_data())); } else { diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 5234a7025..def9d9830 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -628,164 +628,99 @@ kernel void attn_decode_i8( constant uint& head_dim [[buffer(10)]], constant uint& v_hdim [[buffer(11)]], constant uint& history_len [[buffer(12)]], constant float& scale [[buffer(13)]], constant uint& kv_start [[buffer(14)]], constant uint& kv_end [[buffer(15)]], - uint h [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], - uint T [[threads_per_threadgroup]], threadgroup float* sc [[threadgroup(0)]]) + device float* part_o [[buffer(16)]], device float* part_ml [[buffer(17)]], + constant uint& nwg [[buffer(18)]], + uint tg [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]], threadgroup float* smem [[threadgroup(0)]]) { + uint h = tg / nwg, w = tg % nwg; uint kvh = h / (num_q_heads / num_kv_heads); uint ngK = (head_dim + 31u)/32u, ngV = (v_hdim + 31u)/32u; + uint nsg = T / 32u; device const half* qh = q + (size_t)h*head_dim; - threadgroup float* red = sc + kv_end; - float lmax = -INFINITY; - for (uint k = kv_start + t; k < kv_end; k += T) { - float dot = 0; + float qreg[16]; + { uint i=0; for (uint d=lane; d0;s>>=1){ if(t0;s>>=1){ if(t 0 ? 1.0f/red[0] : 0.0f; - threadgroup_barrier(mem_flags::mem_threadgroup); - - for (uint d = t; d < v_hdim; d += T) { - float acc=0; - for (uint k=kv_start; k= cend) { - if (t == 0) { part_m[(size_t)h*C+c] = -INFINITY; part_l[(size_t)h*C+c] = 0.0f; } - for (uint d=t; d 0.0f ? 1.0f/gl : 0.0f; + for (uint d=t; d 0 ? 1.0f/gls[0] : 0.0f; + float gm = -INFINITY; + for (uint w=0;w 0.0f ? 1.0f/gl : 0.0f; for (uint d=t; d psoEmbOm = nil, psoEmbHm = nil; id psoCopy=nil, psoBinary=nil, psoScalar=nil, psoUnary=nil, psoRms=nil, psoSwiglu=nil, psoRmsAdd=nil; id psoCF16F32=nil, psoCF32F16=nil, psoCI8F16=nil, psoCF16I8=nil; - id psoAttn=nil, psoStrided=nil, psoScatter=nil, psoBcast=nil, psoKvAppend=nil; - id psoAttnP=nil, psoAttnC=nil; + id psoAttn=nil, psoAttnC=nil, psoStrided=nil, psoScatter=nil, psoBcast=nil, psoKvAppend=nil; id psoAttnPre=nil, psoAttnPreMma2=nil; id psoKvAppendM=nil, psoKvAppendRingM=nil; id psoSlideS=nil, psoSlideR=nil, psoSlideRM=nil; @@ -71,8 +70,8 @@ psoUnary=pso("unary_f16"); psoRms=pso("rms_norm_f16"); psoSwiglu=pso("swiglu_f16"); psoRmsAdd=pso("rms_norm_add_f16"); psoCF16F32=pso("cast_f16_f32"); psoCF32F16=pso("cast_f32_f16"); psoCI8F16=pso("cast_i8_f16"); psoCF16I8=pso("cast_f16_i8"); - psoAttn=pso("attn_decode_i8"); psoStrided=pso("strided_copy_f16"); psoBcast=pso("bcast_binary_f16"); - psoAttnP=pso("attn_decode_i8_partial"); psoAttnC=pso("attn_decode_combine"); + psoAttn=pso("attn_decode_i8"); psoAttnC=pso("attn_decode_combine"); + psoStrided=pso("strided_copy_f16"); psoBcast=pso("bcast_binary_f16"); psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoKvAppendM=pso("kv_append_i8_m"); psoKvAppendRingM=pso("kv_append_ring_i8_m"); psoSlideS=pso("kv_slide_save"); psoSlideR=pso("kv_slide_restore"); psoSlideRM=pso("kv_slide_restore_m"); @@ -80,7 +79,7 @@ psoArgmax=pso("argmax_logits"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 - &&psoAttn&&psoAttnP&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; + &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; }} }; @@ -648,57 +647,34 @@ bool cactus_metal_encode_attention_i8( setCache(ks, ks_bytes, 5); setCache(vs, vs_bytes, 6); }; - constexpr uint32_t kAttnChunkKeys = 96, kAttnMaxParts = 64; + const uint32_t T = 256, nsg = T / 32u; const uint32_t R = kv_end - kv_start; - uint32_t chunk = kAttnChunkKeys, C = (R + chunk - 1) / chunk; - if (C > kAttnMaxParts) { C = kAttnMaxParts; chunk = (R + C - 1) / C; } - - if (C <= 1) { - const uint32_t T = 256; - [g_enc setComputePipelineState:ctx().psoAttn]; - setInputs(); - setBufAt(out, (size_t)num_q_heads*v_hdim*2, 7); - [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; - [g_enc setBytes:&head_dim length:4 atIndex:10]; [g_enc setBytes:&v_hdim length:4 atIndex:11]; - [g_enc setBytes:&history_len length:4 atIndex:12];[g_enc setBytes:&scale length:4 atIndex:13]; - [g_enc setBytes:&kv_start length:4 atIndex:14]; [g_enc setBytes:&kv_end length:4 atIndex:15]; - [g_enc setThreadgroupMemoryLength:((size_t)kv_end + T)*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; - barrier(); - return true; + uint32_t nwg = (R <= 256u) ? 1u : R / 96u; if (nwg > 32u) nwg = 32u; + id partO = ctx().dummy, partML = ctx().dummy; + if (nwg > 1u) { + partO = recycled((size_t)num_q_heads*nwg*v_hdim*sizeof(float)); + partML = recycled((size_t)num_q_heads*nwg*2*sizeof(float)); } - - uint32_t TP = chunk < 128 ? chunk : 128; - TP = (TP + 31u) & ~31u; - if (TP < 32) TP = 32; - id partO = recycled((size_t)num_q_heads*C*v_hdim*sizeof(float)); - id partM = recycled((size_t)num_q_heads*C*sizeof(float)); - id partL = recycled((size_t)num_q_heads*C*sizeof(float)); - [g_enc setComputePipelineState:ctx().psoAttnP]; + [g_enc setComputePipelineState:ctx().psoAttn]; setInputs(); - [g_enc setBuffer:partO offset:0 atIndex:7]; - [g_enc setBuffer:partM offset:0 atIndex:8]; - [g_enc setBuffer:partL offset:0 atIndex:9]; - [g_enc setBytes:&num_q_heads length:4 atIndex:10]; [g_enc setBytes:&num_kv_heads length:4 atIndex:11]; - [g_enc setBytes:&head_dim length:4 atIndex:12]; [g_enc setBytes:&v_hdim length:4 atIndex:13]; - [g_enc setBytes:&history_len length:4 atIndex:14]; [g_enc setBytes:&scale length:4 atIndex:15]; - [g_enc setBytes:&kv_start length:4 atIndex:16]; [g_enc setBytes:&kv_end length:4 atIndex:17]; - [g_enc setBytes:&chunk length:4 atIndex:18]; [g_enc setBytes:&C length:4 atIndex:19]; - [g_enc setThreadgroupMemoryLength:((size_t)chunk + TP)*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads*C,1,1) threadsPerThreadgroup:MTLSizeMake(TP,1,1)]; - barrier(); - - const uint32_t TC = 256; - [g_enc setComputePipelineState:ctx().psoAttnC]; - [g_enc setBuffer:partO offset:0 atIndex:0]; - [g_enc setBuffer:partM offset:0 atIndex:1]; - [g_enc setBuffer:partL offset:0 atIndex:2]; - setBufAt(out, (size_t)num_q_heads*v_hdim*2, 3); - [g_enc setBytes:&num_q_heads length:4 atIndex:4]; [g_enc setBytes:&v_hdim length:4 atIndex:5]; - [g_enc setBytes:&C length:4 atIndex:6]; - [g_enc setThreadgroupMemoryLength:(size_t)C*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(TC,1,1)]; - barrier(); + setBufAt(out, (size_t)num_q_heads*v_hdim*2, 7); + [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; + [g_enc setBytes:&head_dim length:4 atIndex:10]; [g_enc setBytes:&v_hdim length:4 atIndex:11]; + [g_enc setBytes:&history_len length:4 atIndex:12];[g_enc setBytes:&scale length:4 atIndex:13]; + [g_enc setBytes:&kv_start length:4 atIndex:14]; [g_enc setBytes:&kv_end length:4 atIndex:15]; + [g_enc setBuffer:partO offset:0 atIndex:16]; [g_enc setBuffer:partML offset:0 atIndex:17]; + [g_enc setBytes:&nwg length:4 atIndex:18]; + [g_enc setThreadgroupMemoryLength:((size_t)nsg*v_hdim + 2*nsg)*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + barrier(); + if (nwg > 1u) { + [g_enc setComputePipelineState:ctx().psoAttnC]; + [g_enc setBuffer:partO offset:0 atIndex:0]; [g_enc setBuffer:partML offset:0 atIndex:1]; + setBufAt(out, (size_t)num_q_heads*v_hdim*2, 2); + [g_enc setBytes:&v_hdim length:4 atIndex:3]; [g_enc setBytes:&nwg length:4 atIndex:4]; + [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + } return true; } diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index f0e2e4174..78d14153d 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -345,6 +345,8 @@ def create_parser(): parents=[_build_parent()]) test_parser.add_argument("--component", choices=COMPONENTS, default="all", help="Component to test (default: all)") + test_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", + help="Inference backend, default auto (Metal on Apple GPU, gemma-4-E2B-it only)") test_parser.add_argument("--model", dest="model_id", default=None, type=_hf_id_or_path, help=f"HF model ID under test (default: {DEFAULT_TEST_MODEL_ID})") diff --git a/python/cactus/cli/test.py b/python/cactus/cli/test.py index 8c64ba7ea..82b43fa6f 100644 --- a/python/cactus/cli/test.py +++ b/python/cactus/cli/test.py @@ -31,6 +31,7 @@ def _component_args(component, args): if component == "engine": cmd.extend(["--model", args.model_id]) cmd.extend(["--transcription-model", args.transcription_model_id]) + cmd.extend(["--backend", getattr(args, "backend", "auto")]) if args.android: cmd.append("--android") if args.ios: From 57b3cf01066df8d165c8daab9e41e86f4f165981 Mon Sep 17 00:00:00 2001 From: jakmro Date: Wed, 1 Jul 2026 21:21:51 +0200 Subject: [PATCH 05/36] even faster Signed-off-by: jakmro --- cactus-engine/src/model.cpp | 4 +- cactus-graph/cactus_graph.h | 1 + cactus-graph/src/gpu_fused.cpp | 204 ++++++++++++ cactus-kernels/metal_backend.h | 2 + cactus-kernels/src/cactus_kernels.metal | 358 ++++++++++++++++------ cactus-kernels/src/metal_backend.mm | 114 +++++-- cactus-kernels/src/metal_backend_stub.cpp | 1 + 7 files changed, 579 insertions(+), 105 deletions(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 3ca261987..d41210642 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -1290,7 +1290,9 @@ void Model::execute_prefill_chunk(Component& chunk_comp, Component* enc_comp, si copy_component_outputs_to_chunk_inputs(*encoder_, chunk_comp, i); } } - chunk_comp.graph->execute(); + static const bool fused_prefill = std::getenv("CACTUS_FUSED_PREFILL") != nullptr; + if (!fused_prefill || !chunk_comp.graph->execute_gpu_fused_prefill(static_cast(chunk_tokens))) + chunk_comp.graph->execute(); } void Model::reset_prefill_stats() { diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 22c553f3b..6846179d7 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -722,6 +722,7 @@ class CactusGraph { void execute(const std::string& profile_file = ""); bool execute_gpu_fused(); + bool execute_gpu_fused_prefill(uint32_t M); bool extract_ple_pathway(FusedEmbedCtx& ctx) const; void hard_reset(); void soft_reset(); diff --git a/cactus-graph/src/gpu_fused.cpp b/cactus-graph/src/gpu_fused.cpp index 1cb970e42..9fb39e1ef 100644 --- a/cactus-graph/src/gpu_fused.cpp +++ b/cactus-graph/src/gpu_fused.cpp @@ -295,3 +295,207 @@ bool CactusGraph::execute_gpu_fused() { cactus_metal_session_end(); return true; } + +namespace { +struct PreAttn { size_t window=0, v_hdim=0, po=0; bool causal=true; float scale=0.f; }; +struct GpuModelP { + bool init = false; + LayerW L[NL]; + size_t kc_idx[15], vc_idx[15]; + PreAttn A[NL]; + const void *vnorm, *vnorm_g, *cosL, *sinL, *cosG, *sinG, *final_norm; + CactusQuantMatrix lm_head; + Arena arena; +}; +static GpuModelP GP; +} + +// Fused M>1 prefill: mirrors execute_gpu_fused (decode) for a chunk of M tokens, keeping every +// intermediate on-chip instead of round-tripping the generic graph executor. Weight/norm node +// indices are IDENTICAL to the decode graph (verified). Falls back (returns false) on any KV wrap +// or unsupported shape so the generic path stays authoritative for the hard cases. +bool CactusGraph::execute_gpu_fused_prefill(uint32_t M) { + if (!cactus_metal_available() || M <= 1) return false; + const size_t n = nodes_.size(); + if (n < 3861) return false; + auto B = [&](size_t idx) -> BufferDesc& { return nodes_[idx]->output_buffer; }; + auto P = [&](size_t idx) -> void* { return B(idx).get_data(); }; + if (B(0).total_size != (size_t)M*HID || nodes_[0]->op_type != OpType::INPUT) return false; + g_gpu_argmax_valid = false; + + if (!GP.init) { + auto cq = [&](size_t idx){ return B(idx).to_cq_matrix(); }; + for (int l = 0; l < NL; ++l) { + LayerW& w = GP.L[l]; + w.global = (l % 5 == 4); w.hd = w.global ? 512 : 256; w.shared = (l >= 15); + if (!w.shared) { + size_t b1 = 3 + 10*l, b2 = 334 + 6*l; + w.in_norm=P(b1); w.q_norm=P(b1+1); w.k_norm=P(b1+2); w.post_attn=P(b1+3); + w.pre_ffn=P(b1+4); w.gate=cq(b1+5); w.up=cq(b1+6); w.down=cq(b1+7); + w.post_ffn=P(b1+8); w.post_ple=P(b1+9); + w.q=cq(b2); w.k=cq(b2+1); w.v=cq(b2+2); w.o=cq(b2+3); w.ple_down=cq(b2+4); w.ple_up=cq(b2+5); + w.mlp = 6144; + } else { + int j = l - 15; size_t b1 = 153 + 9*j, b2 = 424 + 4*j; + w.in_norm=P(b1); w.q_norm=P(b1+1); w.post_attn=P(b1+2); w.pre_ffn=P(b1+3); + w.gate=cq(b1+4); w.up=cq(b1+5); w.down=cq(b1+6); w.post_ffn=P(b1+7); w.post_ple=P(b1+8); + w.q=cq(b2); w.o=cq(b2+1); w.ple_down=cq(b2+2); w.ple_up=cq(b2+3); + w.mlp = 12288; + } + w.scalar = P(505 + l); + } + GP.vnorm=P(540); GP.vnorm_g=P(541); GP.cosL=P(542); GP.sinL=P(543); GP.cosG=P(544); GP.sinG=P(545); + GP.final_norm=P(333); GP.lm_head=B(504).to_cq_matrix(); + std::vector cidx; + for (size_t i = 0; i < n && cidx.size() < 30; ++i) + if (nodes_[i]->op_type == OpType::KV_CACHE_STATE) cidx.push_back(i); + if (cidx.size() < 30) return false; + for (int l = 0; l < 15; ++l) { GP.kc_idx[l]=cidx[2*l]; GP.vc_idx[l]=cidx[2*l+1]; } + // per-layer attention params from the graph's ATTENTION_CACHED nodes (in layer order) + int ai = 0; + for (size_t i = 0; i < n && ai < NL; ++i) { + if (nodes_[i]->op_type != OpType::ATTENTION_CACHED) continue; + auto& pr = nodes_[i]->params; + GP.A[ai].window = pr.window_size; GP.A[ai].causal = pr.is_causal; + GP.A[ai].scale = pr.scale; GP.A[ai].v_hdim = pr.v_head_dim; GP.A[ai].po = pr.position_offset; + ai++; + } + if (ai < NL) return false; + GP.init = true; + } + + for (int l = 0; l < 15; ++l) { GP.L[l].kc = P(GP.kc_idx[l]); GP.L[l].vc = P(GP.vc_idx[l]); } + for (int l = 15; l < NL; ++l) { int src = GP.L[l].global ? 14 : 13; GP.L[l].kc = GP.L[src].kc; GP.L[l].vc = GP.L[src].vc; } + for (int l = 0; l < NL; ++l) if (!GP.L[l].kc || !GP.L[l].vc) return false; + + // Chunk start position = tokens already in the cache (before this chunk's append). RoPE rotates + // this chunk's tokens at absolute positions pos0..pos0+M-1. Derive from the KV header (robust to + // however the graph's position input is populated). + int pos0 = (int)((const uint64_t*)GP.L[0].kc)[0]; + // Safe first version: bail (generic handles it) if any owned KV cache would wrap this chunk. + for (int l = 0; l < 15; ++l) { + const uint64_t* km = (const uint64_t*)GP.L[l].kc; + if (km[0] + M > km[1]) return false; // would exceed cache capacity -> ring/sliding path + } + + GP.arena.reset(); + auto fresh = [&](size_t elems){ return GP.arena.fresh(elems * 2); }; + cactus_metal_session_begin(); + cactus_metal_set_active(true); + auto bail = [&](){ cactus_metal_set_active(false); cactus_metal_session_end(); return false; }; + + // gather per-token cos/sin for the M positions (local hd=256, global hd=512) + void* csL = fresh((size_t)M*256); void* snL = fresh((size_t)M*256); + void* csG = fresh((size_t)M*512); void* snG = fresh((size_t)M*512); + for (uint32_t m = 0; m < M; ++m) { + int p = pos0 + (int)m; + std::memcpy((char*)csL+(size_t)m*256*2, (const char*)GP.cosL+(size_t)p*256*2, 256*2); + std::memcpy((char*)snL+(size_t)m*256*2, (const char*)GP.sinL+(size_t)p*256*2, 256*2); + std::memcpy((char*)csG+(size_t)m*512*2, (const char*)GP.cosG+(size_t)p*512*2, 512*2); + std::memcpy((char*)snG+(size_t)m*512*2, (const char*)GP.sinG+(size_t)p*512*2, 512*2); + } + + void* h = fresh((size_t)M*HID); + std::memcpy(h, P(0), (size_t)M*HID*2); + const void* pleBase = P(1); + + for (int l = 0; l < NL; ++l) { + LayerW& w = GP.L[l]; + const int hd = w.hd, QD = NQH*hd; + const void* rc = w.global ? csG : csL; + const void* rs = w.global ? snG : snL; + uint64_t* km = (uint64_t*)w.kc; + size_t max_seq = km[1], kv_heads = km[2], sink = km[4]; + size_t ngK = (hd+31)/32; + PreAttn& AP = GP.A[l]; + float ascale = AP.scale != 0.f ? AP.scale : 1.0f/std::sqrt((float)hd); + + void* xn = fresh((size_t)M*HID); cactus_metal_encode_rms_norm(xn, h, w.in_norm, M, HID, RMS_EPS); + void* q = fresh((size_t)M*QD); if(!cactus_metal_encode_quant_matmul_m(q, xn, &w.q, M)) return bail(); + void* qn = fresh((size_t)M*QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, (size_t)M*NQH, hd, RMS_EPS); + void* qr = fresh((size_t)M*QD); cactus_metal_encode_rope_m(qr, qn, rc, rs, NQH, hd, M); + + size_t hist, newlen; const void *knewp, *vnewp; size_t attn_pos; + if (!w.shared) { + void* k = fresh((size_t)M*hd); if(!cactus_metal_encode_quant_matmul_m(k, xn, &w.k, M)) return bail(); + void* v = fresh((size_t)M*hd); if(!cactus_metal_encode_quant_matmul_m(v, xn, &w.v, M)) return bail(); + void* kn = fresh((size_t)M*hd); cactus_metal_encode_rms_norm(kn, k, w.k_norm, M, hd, RMS_EPS); + void* vn = fresh((size_t)M*hd); cactus_metal_encode_rms_norm(vn, v, w.global ? GP.vnorm_g : GP.vnorm, M, hd, RMS_EPS); + void* kr = fresh((size_t)M*hd); cactus_metal_encode_rope_m(kr, kn, rc, rs, 1, hd, M); + size_t clen = km[0]; + size_t vhd0 = GP.A[l].v_hdim > 0 ? GP.A[l].v_hdim : (size_t)hd; + size_t vmax0 = ((uint64_t*)w.vc)[1], ngV0 = (vhd0+31)/32; + char* kbase = (char*)w.kc; char* vbase = (char*)w.vc; + if (!cactus_metal_encode_kv_append_i8_m(kr, kbase+64, kbase+64+max_seq*kv_heads*hd, + (uint32_t)kv_heads, (uint32_t)hd, (uint32_t)clen, 32, M, + (size_t)M*hd*2, max_seq*kv_heads*hd, max_seq*kv_heads*ngK*sizeof(float))) + return bail(); + if (!cactus_metal_encode_kv_append_i8_m(vn, vbase+64, vbase+64+vmax0*kv_heads*vhd0, + (uint32_t)kv_heads, (uint32_t)vhd0, (uint32_t)clen, 32, M, + (size_t)M*vhd0*2, vmax0*kv_heads*vhd0, vmax0*kv_heads*ngV0*sizeof(float))) + return bail(); + km[0] = clen + M; ((uint64_t*)w.vc)[0] = clen + M; + hist = clen; newlen = M; knewp = kr; vnewp = vn; attn_pos = clen; + } else { + size_t clen = km[0]; + hist = clen; newlen = 0; knewp = nullptr; vnewp = nullptr; + attn_pos = (clen >= M) ? clen - M : 0; + } + + size_t v_hdim = AP.v_hdim > 0 ? AP.v_hdim : (size_t)hd; + size_t v_max = ((uint64_t*)w.vc)[1]; + size_t ngV = (v_hdim+31)/32; + size_t win = AP.window; + uint32_t ringv = (win > 0 && max_seq > 2*sink + 1) ? (uint32_t)(max_seq - 2*sink - 1) : 0u; + char* bk = (char*)w.kc; char* bv = (char*)w.vc; + void* attn = fresh((size_t)M*QD); + if (!cactus_metal_encode_attention_i8_prefill(attn, qr, knewp, vnewp, + bk+64, bv+64, bk+64+max_seq*kv_heads*hd, bv+64+v_max*kv_heads*v_hdim, + (uint32_t)NQH, (uint32_t)kv_heads, (uint32_t)hd, (uint32_t)v_hdim, + (uint32_t)hist, (uint32_t)newlen, (uint32_t)attn_pos, + (uint32_t)win, AP.causal?1u:0u, (uint32_t)M, ascale, + max_seq*kv_heads*hd, v_max*kv_heads*v_hdim, + max_seq*kv_heads*ngK*sizeof(float), v_max*kv_heads*ngV*sizeof(float), + (uint32_t)sink, ringv)) + return bail(); + + void* o = fresh((size_t)M*HID); if(!cactus_metal_encode_quant_matmul_m(o, attn, &w.o, M)) return bail(); + void* h1 = fresh((size_t)M*HID); cactus_metal_encode_rms_norm_add(h1, o, w.post_attn, h, M, HID, RMS_EPS); h = h1; + + const int MLP = w.mlp; + void* xn2 = fresh((size_t)M*HID); cactus_metal_encode_rms_norm(xn2, h, w.pre_ffn, M, HID, RMS_EPS); + void* gate= fresh((size_t)M*MLP); if(!cactus_metal_encode_quant_matmul_m(gate, xn2, &w.gate, M)) return bail(); + void* up = fresh((size_t)M*MLP); if(!cactus_metal_encode_quant_matmul_m(up, xn2, &w.up, M)) return bail(); + void* g3 = fresh((size_t)M*MLP); cactus_metal_encode_swiglu(g3, gate, up, (size_t)M*MLP, GATE_SCALE); + void* mo = fresh((size_t)M*HID); if(!cactus_metal_encode_quant_matmul_m(mo, g3, &w.down, M)) return bail(); + void* h2 = fresh((size_t)M*HID); cactus_metal_encode_rms_norm_add(h2, mo, w.post_ffn, h, M, HID, RMS_EPS); h = h2; + + // PLE: gather ple[*, l, *] (strided by NL*PLE_DIM) into a contiguous [M, PLE_DIM] slice + void* ps = fresh((size_t)M*PLE_DIM); if(!cactus_metal_encode_quant_matmul_m(ps, h, &w.ple_down, M)) return bail(); + void* pled= fresh((size_t)M*PLE_DIM); + { uint32_t osh[2]={M,(uint32_t)PLE_DIM}; uint32_t sst[2]={(uint32_t)(NL*PLE_DIM),1}; + if(!cactus_metal_encode_strided_copy(pled, pleBase, osh, sst, 2, M*PLE_DIM, (uint32_t)(l*PLE_DIM), + (size_t)M*NL*PLE_DIM*2, (size_t)M*PLE_DIM*2)) return bail(); } + void* pm = fresh((size_t)M*PLE_DIM); cactus_metal_encode_swiglu(pm, ps, pled, (size_t)M*PLE_DIM, 1.0f); + void* pu = fresh((size_t)M*HID); if(!cactus_metal_encode_quant_matmul_m(pu, pm, &w.ple_up, M)) return bail(); + void* h3 = fresh((size_t)M*HID); cactus_metal_encode_rms_norm_add(h3, pu, w.post_ple, h, M, HID, RMS_EPS); h = h3; + + float ls = (float)(*(const __fp16*)w.scalar); + void* h4 = fresh((size_t)M*HID); cactus_metal_encode_scalar(2, h4, h, (size_t)M*HID, ls); h = h4; + } + + // logits for the LAST token only (prepare_decode needs just that one) + const void* hlast = (const char*)h + (size_t)(M-1)*HID*2; + size_t V = B(n-1).total_size; + void* fn = fresh(HID); cactus_metal_encode_rms_norm(fn, hlast, GP.final_norm, 1, HID, RMS_EPS); + void* code = fresh(HID); + void* lg = fresh(V); + if (!cactus_metal_encode_quant_matmul_ortho(lg, fn, code, &GP.lm_head)) return bail(); + cactus_metal_encode_scalar(3, lg, lg, V, SOFTCAP); + cactus_metal_encode_unary(1, lg, lg, V); + cactus_metal_encode_scalar(2, lg, lg, V, SOFTCAP); + B(n-1).set_external(lg); + cactus_metal_set_active(false); + cactus_metal_session_end(); + return true; +} diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index e622fed6f..10ab0e5f5 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -41,6 +41,8 @@ bool cactus_metal_encode_unary(int op_type, void* out, const void* in, size_t n) bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, size_t n, float scale); bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const void* sin, uint32_t heads, uint32_t head_dim); +bool cactus_metal_encode_rope_m(void* out, const void* x, const void* cos_m, const void* sin_m, + uint32_t heads, uint32_t head_dim, uint32_t M); bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps); bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* weight, const void* res, diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index def9d9830..e310f4bbe 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -199,101 +199,63 @@ kernel void cq4_gemm_mma( uint tl [[thread_index_in_threadgroup]], uint sg [[simdgroup_index_in_threadgroup]]) { - const uint ldA = 40u, ldB = 40u; - threadgroup half As0[64u*40u]; - threadgroup half Bs0[32u*40u]; - threadgroup float Cs[64u*32u]; + // llama.cpp-style SG_MAT gemm (ported from ggml-metal kernel_mul_mm, non-tensor path): shared tiles + // are stored as CONTIGUOUS 8x8 blocks so simdgroup_load is conflict-free (vs row-major ldA=40 strided). + // Weight is dequantized (CQ4 codebook) into sa; activation (code) into sb. The fill also transposes + // weight [N,K]->[K,N] via the ly/lx block indexing. Tile 64 N-cols x 32 M-rows x 32 K, 128 threads + // (4 simdgroups), mc[8] accumulators. ~2x faster than the old 64x32 row-major kernel (dense microbench). + threadgroup float pool[2048]; // 8KB: sa+sb (6KB) during K-loop, Cs (8KB) at store threadgroup float cb[16]; - threadgroup half* As1 = (threadgroup half*)Cs; - threadgroup half* Bs1 = As1 + 64u*40u; + threadgroup half* sa = (threadgroup half*)pool; // weight 64Nx32K SG_MAT = 2048 halves (4KB) + threadgroup half* sb = sa + 2048u; // activ. 32Mx32K SG_MAT = 1024 halves (2KB) + threadgroup float* Cs = pool; // 32M x 64N float (8KB), aliases sa+sb at store if (tl<16u) cb[tl]=(float)codebook[tl]; - threadgroup_barrier(mem_flags::mem_threadgroup); + const uint NK=32u, NL0=2u, NL1=4u; uint K = num_groups*gs; - uint m0 = tg.y*64u, n0 = tg.x*32u; - uint rb = sg*16u; - simdgroup_matrix C00=make_filled_simdgroup_matrix(0.f); - simdgroup_matrix C01=make_filled_simdgroup_matrix(0.f); - simdgroup_matrix C02=make_filled_simdgroup_matrix(0.f); - simdgroup_matrix C03=make_filled_simdgroup_matrix(0.f); - simdgroup_matrix C10=make_filled_simdgroup_matrix(0.f); - simdgroup_matrix C11=make_filled_simdgroup_matrix(0.f); - simdgroup_matrix C12=make_filled_simdgroup_matrix(0.f); - simdgroup_matrix C13=make_filled_simdgroup_matrix(0.f); - threadgroup half* Ac=As0; threadgroup half* Bc=Bs0; - threadgroup half* An=As1; threadgroup half* Bn=Bs1; - { - uint g = 0u, e0 = 0u; - for (uint i=tl; i<64u*32u; i+=128u){ - uint r=i>>5, k=i&31u; - Ac[r*ldA+k] = (m0+r>5, col=i&31u, n=n0+col; - float v=0.f; - if (n>1)]; - uint nib=(e&1u)?(uint)(by>>4):(uint)(by&0xF); - v=(float)norms[(size_t)n*num_groups+g]*cb[nib]; - } - Bc[k*ldB+col]=(half)v; - } - } + uint r0 = tg.y*64u; // N tile (output features) + uint r1 = tg.x*32u; // M tile (tokens) + uint lr0 = tl/NL0, lr1 = tl/NL1, il0 = tl%NL0; + uint nrow = r0+lr0; + uint iy = 8u*(tl%NL1); + device const half* yrow = code + (size_t)(r1+lr1)*K; + simdgroup_matrix ma[4], mb[2]; + simdgroup_matrix mc[8]; + for (uint i=0;i<8u;i++) mc[i]=make_filled_simdgroup_matrix(0.f); threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint k0=0; k0>5, k=i&31u; - An[r*ldA+k] = (m0+r>5, col=i&31u, n=n0+col; - float v=0.f; - if (n>1)]; - uint nib=(e&1u)?(uint)(by>>4):(uint)(by&0xF); - v=(float)norms[(size_t)n*num_groups+g]*cb[nib]; - } - Bn[k*ldB+col]=(half)v; - } + for (uint loop_k=0; loop_k>1)]; + uint nib=(e&1u)?(uint)(by>>4):(uint)(by&0xF); v=nrm*cb[nib]; } + sa[64u*ib+8u*ly+lx]=(half)v; } - for (uint kk=0; kk<32u; kk+=8u){ - simdgroup_matrix A0,A1,B0,B1,B2,B3; - simdgroup_load(A0,&Ac[(rb)*ldA+kk],ldA); - simdgroup_load(A1,&Ac[(rb+8u)*ldA+kk],ldA); - simdgroup_load(B0,&Bc[kk*ldB+0u],ldB); - simdgroup_load(B1,&Bc[kk*ldB+8u],ldB); - simdgroup_load(B2,&Bc[kk*ldB+16u],ldB); - simdgroup_load(B3,&Bc[kk*ldB+24u],ldB); - simdgroup_multiply_accumulate(C00,A0,B0,C00); - simdgroup_multiply_accumulate(C01,A0,B1,C01); - simdgroup_multiply_accumulate(C02,A0,B2,C02); - simdgroup_multiply_accumulate(C03,A0,B3,C03); - simdgroup_multiply_accumulate(C10,A1,B0,C10); - simdgroup_multiply_accumulate(C11,A1,B1,C11); - simdgroup_multiply_accumulate(C12,A1,B2,C12); - simdgroup_multiply_accumulate(C13,A1,B3,C13); + for (uint i=0;i<8u;i++){ + uint sx=tl%NL1, sy=(tl/NL1)/8u, lx=i, ly=(tl/NL1)%8u, ib=4u*sx+sy; + sb[64u*ib+8u*ly+lx]=(loop_k+iy+i>1u), n_sg = 32u*(sg&1u); + for (uint i=0;i<8u;i++) + simdgroup_store(mc[i], &Cs[(m_sg+8u*(i/4u))*64u + n_sg+8u*(i%4u)], 64u); threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint i=tl; i<64u*32u; i+=128u){ - uint r=i>>5, c=i&31u; - if (m0+r=total) return; + uint m=gid/per, r=gid%per, d=r%hd, hh=hd/2; + uint base=m*per + (r/hd)*hd; // start of this (token,head) + float rot = (d> 5, lane = tl & 31u; + uint h = sg, kvh = 0u; + uint m0 = tgx*QB; + uint total_keys = history_len + new_len; + + // Sliding-window (ring buffer + attention sink) — mirrors attn_prefill_i8. Attend sink [0,S) plus the + // recent `R` keys; physical cache slot wraps for keys past the window. When the whole block is within + // the window (pos_first < Wsr), iterate all keys; otherwise iterate only sink + the recent union. + uint S = (ringR > 0u) ? sinkN : 0u; + uint R = ringR, Wsr = S + R; + uint pos_first = q_pos0 + m0; + uint pos_last = q_pos0 + m0 + (QB-1u); + uint kv_end_tile = min(total_keys, pos_last + 1u); + bool windowed = (R > 0u && pos_first >= Wsr); + uint rstart = windowed ? (pos_first + 1u - R) : 0u; + uint nactive = windowed ? (S + (kv_end_tile - rstart)) : kv_end_tile; + simdgroup_matrix O[NFH]; + + for (uint f=0; f(0.f); + if (lane < QB) { mrun[h*QB+lane] = -INFINITY; lrun[h*QB+lane]=0.0f; } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint k0=0; k0 C0=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C1=C0,C2=C0,C3=C0,C4=C0,C5=C0,C6=C0,C7=C0; + for (uint c=0; c0u && kk>=Wsr)?(S+(kk-S)%R):kk; + device const char* kp=kc+((size_t)slot*num_kv_heads+kvh)*HD; + device const float* sp=ks+((size_t)slot*num_kv_heads+kvh)*(HD/32u); + val=(half)((float)kp[c+d]*sp[(c+d)/32u]); + } else { + device const half* kp=knew+((size_t)(kk-history_len)*num_kv_heads+kvh)*HD; + val=kp[c+d]; + } + } + Ks[d*LDK + k]=val; + } + for (uint i=lane; i>5, d=i&31u; + Qs[h*QB*LD + r*LD + d] = (m0+r A,B0,B1,B2,B3,B4,B5,B6,B7; + simdgroup_load(A, &Qs[h*QB*LD + kk], LD); + simdgroup_load(B0,&Ks[kk*LDK + 0u], LDK); + simdgroup_load(B1,&Ks[kk*LDK + 8u], LDK); + simdgroup_load(B2,&Ks[kk*LDK + 16u],LDK); + simdgroup_load(B3,&Ks[kk*LDK + 24u],LDK); + simdgroup_load(B4,&Ks[kk*LDK + 32u],LDK); + simdgroup_load(B5,&Ks[kk*LDK + 40u],LDK); + simdgroup_load(B6,&Ks[kk*LDK + 48u],LDK); + simdgroup_load(B7,&Ks[kk*LDK + 56u],LDK); + simdgroup_multiply_accumulate(C0,A,B0,C0); + simdgroup_multiply_accumulate(C1,A,B1,C1); + simdgroup_multiply_accumulate(C2,A,B2,C2); + simdgroup_multiply_accumulate(C3,A,B3,C3); + simdgroup_multiply_accumulate(C4,A,B4,C4); + simdgroup_multiply_accumulate(C5,A,B5,C5); + simdgroup_multiply_accumulate(C6,A,B6,C6); + simdgroup_multiply_accumulate(C7,A,B7,C7); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + simdgroup_store(C0,&Ss[h*QB*BK + 0u], BK); + simdgroup_store(C1,&Ss[h*QB*BK + 8u], BK); + simdgroup_store(C2,&Ss[h*QB*BK + 16u],BK); + simdgroup_store(C3,&Ss[h*QB*BK + 24u],BK); + simdgroup_store(C4,&Ss[h*QB*BK + 32u],BK); + simdgroup_store(C5,&Ss[h*QB*BK + 40u],BK); + simdgroup_store(C6,&Ss[h*QB*BK + 48u],BK); + simdgroup_store(C7,&Ss[h*QB*BK + 56u],BK); + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (lane < QB) { + uint r=lane, qpos=q_pos0+m0+r; + threadgroup float* srow=&Ss[h*QB*BK + r*BK]; + float tmax=-INFINITY; + for (uint k=0;kqpos); + float s = att ? srow[k]*scale : -INFINITY; + srow[k]=s; tmax=max(tmax,s); + } + float mo=mrun[h*QB+r], mnew=max(mo,tmax); + float resc=exp(mo-mnew), tsum=0.0f; + for (uint k=0;k-INFINITY)?exp(srow[k]-mnew):0.0f; + srow[k]=e; tsum+=e; + } + lrun[h*QB+r]=lrun[h*QB+r]*resc+tsum; + mrun[h*QB+r]=mnew; resc8[h*QB+r]=resc; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (lane == 0u) { + int nr = 0; + for (uint r=0; r>3; Rsc[sg*QB*8u + e] *= resc8[h*QB+r]; } + simdgroup_barrier(mem_flags::mem_threadgroup); + simdgroup_load(O[f], &Rsc[sg*QB*8u], 8u); + simdgroup_barrier(mem_flags::mem_threadgroup); + } + } + + for (uint i=lane; i>5, d=i&31u; + half val=(half)0; + if (k0u && kk>=Wsr)?(S+(kk-S)%R):kk; + device const char* vp=vc+((size_t)slot*num_kv_heads+kvh)*VD; + device const float* sp=vs+((size_t)slot*num_kv_heads+kvh)*(VD/32u); + val=(half)((float)vp[c+d]*sp[(c+d)/32u]); + } else { + device const half* vp=vnew+((size_t)(kk-history_len)*num_kv_heads+kvh)*VD; + val=vp[c+d]; + } + } + Vs[k*LD + d]=val; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint fc=0; fc<32u; fc+=8u){ + uint f = jc/8u + fc/8u; + for (uint kk=0; kk P,V0; + simdgroup_load(P, &Ps[h*QB*LDP + kk], LDP); + simdgroup_load(V0, &Vs[kk*LD + fc], LD); + simdgroup_multiply_accumulate(O[f], P, V0, O[f]); + } + } + } + } + for (uint f=0; f>3, d=e&7u; + if (m0+r0?1.0f/lrun[h*QB+r]:0.0f; + out[((size_t)(m0+r)*num_q_heads + h)*VD + (f*8u + d)] = (half)(Rsc[sg*QB*8u + e]*inv); + } + } + simdgroup_barrier(mem_flags::mem_threadgroup); + } +} + kernel void kv_append_i8_m(device const half* src [[buffer(0)]], device char* int8base [[buffer(1)]], device float* scalebase [[buffer(2)]], constant uint& kv_heads [[buffer(3)]], constant uint& hdim [[buffer(4)]], constant uint& current_len [[buffer(5)]], diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 7fe33b344..ecc7cc466 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -35,10 +35,10 @@ id psoCopy=nil, psoBinary=nil, psoScalar=nil, psoUnary=nil, psoRms=nil, psoSwiglu=nil, psoRmsAdd=nil; id psoCF16F32=nil, psoCF32F16=nil, psoCI8F16=nil, psoCF16I8=nil; id psoAttn=nil, psoAttnC=nil, psoStrided=nil, psoScatter=nil, psoBcast=nil, psoKvAppend=nil; - id psoAttnPre=nil, psoAttnPreMma2=nil; + id psoAttnPre=nil, psoAttnPreMma2=nil, psoAttnPreHd256=nil; id psoKvAppendM=nil, psoKvAppendRingM=nil; id psoSlideS=nil, psoSlideR=nil, psoSlideRM=nil; - id psoRope=nil; + id psoRope=nil, psoRopeM=nil; id psoArgmax=nil; id psoGather=nil; id dummy=nil; @@ -72,14 +72,14 @@ psoCI8F16=pso("cast_i8_f16"); psoCF16I8=pso("cast_f16_i8"); psoAttn=pso("attn_decode_i8"); psoAttnC=pso("attn_decode_combine"); psoStrided=pso("strided_copy_f16"); psoBcast=pso("bcast_binary_f16"); - psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoKvAppendM=pso("kv_append_i8_m"); + psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoAttnPreHd256=pso("attn_prefill_mma_hd256"); psoKvAppendM=pso("kv_append_i8_m"); psoKvAppendRingM=pso("kv_append_ring_i8_m"); psoSlideS=pso("kv_slide_save"); psoSlideR=pso("kv_slide_restore"); psoSlideRM=pso("kv_slide_restore_m"); - psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); + psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); psoRopeM=pso("rope_f16_m"); psoArgmax=pso("argmax_logits"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 - &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; + &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoRopeM&&psoArgmax&&psoGather; }} }; @@ -155,6 +155,29 @@ void deinterleave_4row(const CactusQuantMatrix* W, inline size_t bucket(size_t b) { return (b + 4095) & ~size_t(4095); } +struct GemmProf { + double gpu_ns=0; long cmdbufs=0; // non-perturbing: total GPU-busy across all committed command buffers + ~GemmProf(){ if(std::getenv("CACTUS_PROF_GPU") && cmdbufs>0) + fprintf(stderr,"[PROF gpu-busy] cmdbufs=%ld total_gpu=%.2fms\n", cmdbufs, gpu_ns/1e6); } +}; +static GemmProf g_gemmprof; +static const bool g_prof_gpu = (std::getenv("CACTUS_PROF_GPU") != nullptr); + +// Per-op-type GPU-time profiler: flush+time each dispatch, attribute to the current op tag g_op. +// Perturbing (serializes), but gives an exact per-category GPU-time breakdown of a forward pass. +static const bool g_prof_ops = (std::getenv("CACTUS_PROF_OPS") != nullptr); +static const bool g_skip_mm=(std::getenv("CACTUS_SKIP_MM")!=nullptr), g_skip_attn=(std::getenv("CACTUS_SKIP_ATTN")!=nullptr), g_skip_norm=(std::getenv("CACTUS_SKIP_NORM")!=nullptr); +static const char* g_op = "other"; +struct OpProf { + std::map> t; + ~OpProf(){ if(!g_prof_ops || t.empty()) return; + double tot=0; for(auto&kv:t) tot+=kv.second.first; + fprintf(stderr,"[PROF ops] total=%.1fms\n", tot/1e6); + for(auto&kv:t) fprintf(stderr," %-12s %8.2fms %5.1f%% (n=%ld)\n", + kv.first.c_str(), kv.second.first/1e6, 100.0*kv.second.first/tot, kv.second.second); } +}; +static OpProf g_opprof; + static const bool g_concurrent = (std::getenv("CACTUS_GPU_CONCURRENT") != nullptr); id ensureEncoder() { @@ -168,6 +191,16 @@ void deinterleave_4row(const CactusQuantMatrix* W, inline void barrier() { static const bool force = (std::getenv("CACTUS_FORCE_BARRIER") != nullptr); + if (g_prof_ops && g_enc) { // flush + time this single dispatch, attribute to g_op + [g_enc endEncoding]; [g_cmd commit]; [g_cmd waitUntilCompleted]; + g_opprof.t[g_op].first += (g_cmd.GPUEndTime - g_cmd.GPUStartTime)*1e9; + g_opprof.t[g_op].second += 1; + g_enc = nil; g_cmd = nil; + for (id b : g_pending) g_free[(size_t)b.length].push_back(b); + g_pending.clear(); + g_op = "other"; + return; + } if (g_concurrent || force) [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; } @@ -215,7 +248,10 @@ inline void setBufAt(const void* p, size_t bytes, int idx) { void cactus_metal_session_begin() {} void cactus_metal_session_sync() { if (g_enc) { - [g_enc endEncoding]; [g_cmd commit]; [g_cmd waitUntilCompleted]; + [g_enc endEncoding]; + if (g_prof_gpu) [g_cmd addCompletedHandler:^(id cb){ + g_gemmprof.gpu_ns += (cb.GPUEndTime-cb.GPUStartTime)*1e9; g_gemmprof.cmdbufs += 1; }]; + [g_cmd commit]; [g_cmd waitUntilCompleted]; g_enc = nil; g_cmd = nil; } for (id b : g_pending) g_free[(size_t)b.length].push_back(b); @@ -289,6 +325,7 @@ bool cactus_metal_encode_unary(int op, void* out, const void* in, size_t n) { bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, size_t n, float scale) { if (!ctx().ok) return false; ensureEncoder(); + g_op="swiglu"; uint32_t nn=(uint32_t)n; float s=scale; [g_enc setComputePipelineState:ctx().psoSwiglu]; setBufAt(gate, n*2, 0); setBufAt(up, n*2, 1); setBufAt(out, n*2, 2); @@ -301,6 +338,7 @@ bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const v uint32_t heads, uint32_t head_dim) { if (!ctx().ok) return false; ensureEncoder(); + g_op="rope"; uint32_t total=heads*head_dim; [g_enc setComputePipelineState:ctx().psoRope]; setBufAt(x, (size_t)total*2, 0); setBufAt(out, (size_t)total*2, 1); @@ -310,16 +348,31 @@ bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const v barrier(); return true; } +bool cactus_metal_encode_rope_m(void* out, const void* x, const void* cos_m, const void* sin_m, + uint32_t heads, uint32_t head_dim, uint32_t M) { + if (!ctx().ok) return false; + ensureEncoder(); + g_op="rope"; + uint32_t total=M*heads*head_dim; + [g_enc setComputePipelineState:ctx().psoRopeM]; + setBufAt(x, (size_t)total*2, 0); setBufAt(out, (size_t)total*2, 1); + setBufAt(cos_m, (size_t)M*head_dim*2, 2); setBufAt(sin_m, (size_t)M*head_dim*2, 3); + [g_enc setBytes:&heads length:4 atIndex:4]; [g_enc setBytes:&head_dim length:4 atIndex:5]; [g_enc setBytes:&M length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps) { if (!ctx().ok) return false; ensureEncoder(); + g_op="rmsnorm"; uint32_t d=(uint32_t)dim; float e=eps; [g_enc setComputePipelineState:ctx().psoRms]; setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(out, rows*dim*2, 2); [g_enc setBytes:&d length:4 atIndex:3]; [g_enc setBytes:&e length:4 atIndex:4]; [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + if(!g_skip_norm) [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; barrier(); return true; } @@ -327,18 +380,20 @@ bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* wei size_t rows, size_t dim, float eps) { if (!ctx().ok) return false; ensureEncoder(); + g_op="rmsnorm"; uint32_t d=(uint32_t)dim; float e=eps; [g_enc setComputePipelineState:ctx().psoRmsAdd]; setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(res, rows*dim*2, 2); setBufAt(out, rows*dim*2, 3); [g_enc setBytes:&d length:4 atIndex:4]; [g_enc setBytes:&e length:4 atIndex:5]; [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + if(!g_skip_norm) [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; barrier(); return true; } bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) { if (!ctx().ok) return false; ensureEncoder(); + g_op="argmax"; uint32_t V = vocab, T = 1024; [g_enc setComputePipelineState:ctx().psoArgmax]; setBufAt(logits, (size_t)vocab*2, 0); @@ -392,6 +447,7 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu if (!g_code_buf || (size_t)g_code_buf.length < code_bytes) g_code_buf = [ctx().dev newBufferWithLength:code_bytes options:MTLResourceStorageModeShared]; ensureEncoder(); + g_op="matmul"; bool simdT = (gs==128u && ctx().psoTsimd); [g_enc setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; setBufAt(lhs, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; @@ -407,7 +463,8 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; uint32_t ROWS=8; - [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + g_op="matmul"; + if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; barrier(); return true; } @@ -429,6 +486,7 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus if (!g_code_buf_m || (size_t)g_code_buf_m.length < code_bytes) g_code_buf_m = [ctx().dev newBufferWithLength:code_bytes options:MTLResourceStorageModeShared]; ensureEncoder(); + g_op="matmul"; [g_enc setComputePipelineState:ctx().psoTm]; setBufAt(lhs, (size_t)M*K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; @@ -436,6 +494,7 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus [g_enc setBuffer:rw.perm offset:0 atIndex:4]; [g_enc setBuffer:g_code_buf_m offset:0 atIndex:5]; [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setBytes:&K length:4 atIndex:7]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; + g_op="transform"; [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; barrier(); @@ -446,12 +505,13 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus setBufAt(out, (size_t)M*N*2, 4); [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; [g_enc setBytes:&M length:4 atIndex:9]; - if (mma) { - [g_enc dispatchThreadgroups:MTLSizeMake((N+31)/32,(M+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + g_op="matmul"; + if (!g_skip_mm) { if (mma) { + [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; } else { uint32_t ROWS=8, MT=16; [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,(M+MT-1)/MT,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; - } + } } barrier(); return true; } @@ -479,7 +539,7 @@ bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* co [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; uint32_t ROWS=8; - [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; barrier(); return true; } @@ -630,6 +690,7 @@ bool cactus_metal_encode_attention_i8( if (!ctx().ok) return false; if (kv_end <= kv_start || (num_q_heads % num_kv_heads) != 0) return false; ensureEncoder(); + g_op="attention"; auto setCache = [&](const void* p, size_t bytes, int idx) { if (p && bytes) setBufAt(p, bytes, idx); @@ -665,7 +726,7 @@ bool cactus_metal_encode_attention_i8( [g_enc setBuffer:partO offset:0 atIndex:16]; [g_enc setBuffer:partML offset:0 atIndex:17]; [g_enc setBytes:&nwg length:4 atIndex:18]; [g_enc setThreadgroupMemoryLength:((size_t)nsg*v_hdim + 2*nsg)*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + if(!g_skip_attn)[g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; barrier(); if (nwg > 1u) { [g_enc setComputePipelineState:ctx().psoAttnC]; @@ -690,10 +751,21 @@ bool cactus_metal_encode_attention_i8_prefill( const uint32_t T = 128; if (total_keys == 0 || M == 0 || (num_q_heads % num_kv_heads) != 0) return false; uint32_t maxsc = (ring > 0u) ? ((total_keys > sink + ring) ? (sink + ring) : total_keys) : 256u; + static const bool en_hd256 = (std::getenv("CACTUS_HD256") != nullptr); bool mma2 = (ring == 0u && is_causal && total_keys >= 2048u && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u); - if (mma2) { + // Route ALL global (hd=512, full-attention / window==0) layers through the tiled mma2 kernel, not just + // the long-context ones — the tiled kernel loads each KV tile into shared once and reuses it across a + // block of 8 queries, vs the scalar path re-reading the whole KV cache per query. Always correct for + // window==0 (no sliding-window eviction). Biggest prefill attention win (pp2048 +22%). + if (!mma2 && is_causal && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u + && window == 0u) mma2 = true; + // hd256 now handles the sliding-window (sink + ring slot mapping), so it's valid for local layers too. + bool hd256 = (en_hd256 && ctx().psoAttnPreHd256 != nil && is_causal && head_dim == 256u && v_hdim == 256u + && num_q_heads == 8u && num_kv_heads == 1u); + if (mma2 || hd256) { ensureEncoder(); - [g_enc setComputePipelineState:ctx().psoAttnPreMma2]; + g_op="attention"; + [g_enc setComputePipelineState:(hd256 ? ctx().psoAttnPreHd256 : ctx().psoAttnPreMma2)]; setBufAt(q, (size_t)M*num_q_heads*head_dim*2, 0); if (new_len > 0 && knew && vnew) { setBufAt(knew, (size_t)new_len*num_kv_heads*head_dim*2, 1); @@ -711,8 +783,9 @@ bool cactus_metal_encode_attention_i8_prefill( [g_enc setBytes:&history_len length:4 atIndex:12];[g_enc setBytes:&scale length:4 atIndex:13]; [g_enc setBytes:&q_pos0 length:4 atIndex:14]; [g_enc setBytes:&new_len length:4 atIndex:15]; [g_enc setBytes:&M length:4 atIndex:16]; + if (hd256) { [g_enc setBytes:&sink length:4 atIndex:17]; [g_enc setBytes:&ring length:4 atIndex:18]; } uint32_t mtiles = (M + 7u)/8u; - [g_enc dispatchThreadgroups:MTLSizeMake(mtiles, 1, 1) threadsPerThreadgroup:MTLSizeMake(512,1,1)]; + if(!g_skip_attn)[g_enc dispatchThreadgroups:MTLSizeMake(mtiles, 1, 1) threadsPerThreadgroup:MTLSizeMake(hd256?256u:512u,1,1)]; barrier(); return true; } @@ -738,7 +811,7 @@ bool cactus_metal_encode_attention_i8_prefill( [g_enc setBytes:&maxsc length:4 atIndex:18]; [g_enc setBytes:&sink length:4 atIndex:19]; [g_enc setBytes:&ring length:4 atIndex:20]; [g_enc setThreadgroupMemoryLength:((size_t)maxsc + T)*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(M*num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + if(!g_skip_attn)[g_enc dispatchThreadgroups:MTLSizeMake(M*num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; barrier(); return true; } @@ -787,6 +860,7 @@ bool cactus_metal_encode_kv_append_i8(const void* src, void* int8base, void* sca if (!ctx().ok) return false; uint32_t num_groups = (hdim + group_size - 1)/group_size, n = kv_heads*num_groups; ensureEncoder(); + g_op="kv"; [g_enc setComputePipelineState:ctx().psoKvAppend]; setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; @@ -802,6 +876,7 @@ bool cactus_metal_encode_kv_append_i8_m(const void* src, void* int8base, void* s if (!ctx().ok) return false; uint32_t num_groups = (hdim + group_size - 1)/group_size, n = M*kv_heads*num_groups; ensureEncoder(); + g_op="kv"; [g_enc setComputePipelineState:ctx().psoKvAppendM]; setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; @@ -818,6 +893,7 @@ bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, vo if (!ctx().ok) return false; uint32_t num_groups = (hdim + group_size - 1)/group_size, n = M*kv_heads*num_groups; ensureEncoder(); + g_op="kv"; [g_enc setComputePipelineState:ctx().psoKvAppendRingM]; setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; @@ -837,6 +913,7 @@ bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, v id scrI8 = recycled((size_t)(remaining?remaining:1)*kv_heads*hdim); id scrSc = recycled((size_t)(remaining?remaining:1)*kv_heads*num_groups*sizeof(float)); ensureEncoder(); + g_op="kv"; if (remaining > 0) { uint32_t n = remaining*per; [g_enc setComputePipelineState:ctx().psoSlideS]; @@ -868,6 +945,7 @@ bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, id scrI8 = recycled((size_t)(remaining?remaining:1)*kv_heads*hdim); id scrSc = recycled((size_t)(remaining?remaining:1)*kv_heads*num_groups*sizeof(float)); ensureEncoder(); + g_op="kv"; if (remaining > 0) { uint32_t n = remaining*per; [g_enc setComputePipelineState:ctx().psoSlideS]; diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 8e54eb5d1..9c7ce4407 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -22,6 +22,7 @@ bool cactus_metal_encode_scalar(int, void*, const void*, size_t, float) { return bool cactus_metal_encode_unary(int, void*, const void*, size_t) { return false; } bool cactus_metal_encode_swiglu(void*, const void*, const void*, size_t, float) { return false; } bool cactus_metal_encode_rope(void*, const void*, const void*, const void*, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_rope_m(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_rms_norm(void*, const void*, const void*, size_t, size_t, float) { return false; } bool cactus_metal_encode_rms_norm_add(void*, const void*, const void*, const void*, size_t, size_t, float) { return false; } bool cactus_metal_encode_argmax(const void*, uint32_t, void*) { return false; } From cba8a7418818bd24d8b5d7e5da41c718c0cd4a2e Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 01:38:40 +0200 Subject: [PATCH 06/36] converter Signed-off-by: jakmro --- cactus-engine/src/model.cpp | 21 ++ cactus-graph/src/gpu_fused.cpp | 49 +++-- cactus-kernels/metal_backend.h | 7 + cactus-kernels/src/cactus_kernels.metal | 229 ++++++++++++++++++++++ cactus-kernels/src/metal_backend.mm | 147 ++++++++++++-- cactus-kernels/src/metal_backend_stub.cpp | 4 + python/cactus/cli/runtime.py | 1 + python/cactus/transpile/model_adapters.py | 164 +++++++++++++++- 8 files changed, 591 insertions(+), 31 deletions(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index d41210642..6f2352b1f 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -602,6 +602,27 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, bound.insert(comp); } + // Prewarm the prefill executor. The generic graph executor allocates every intermediate GPU + // buffer (gpu_persistent_acts_) lazily on the FIRST execute() — a ~190ms one-time cost that + // otherwise lands in the first real prefill chunk, inflating TTFT (dominant at short prompts and + // the reason pp512 measured slower than pp2048). Run each prefill graph once on zeroed inputs + // here, then reset the KV cache states the warmup dirtied so real inference starts clean. + if (decoder_prefill_ && decoder_prefill_->graph + && std::getenv("CACTUS_NO_PREFILL_WARM") == nullptr) { + try { + if (prefill_encoder_ && prefill_encoder_->graph) { + for (auto& b : prefill_encoder_->input_buffers) std::fill(b.begin(), b.end(), 0); + prefill_encoder_->graph->execute(); + reset_component_cache_states(*prefill_encoder_); + } + for (auto& b : decoder_prefill_->input_buffers) std::fill(b.begin(), b.end(), 0); + decoder_prefill_->graph->execute(); + reset_component_cache_states(*decoder_prefill_); + } catch (const std::exception& e) { + CACTUS_LOG_WARN("model", std::string("prefill warmup skipped: ") + e.what()); + } + } + if (vision_encoder_ && tokenizer_ && !vision_encoder_->output_node_ids.empty()) { size_t out_node = static_cast(vision_encoder_->output_node_ids[0]); const auto& desc = vision_encoder_->graph->get_output_buffer(out_node); diff --git a/cactus-graph/src/gpu_fused.cpp b/cactus-graph/src/gpu_fused.cpp index 9fb39e1ef..6f527a0aa 100644 --- a/cactus-graph/src/gpu_fused.cpp +++ b/cactus-graph/src/gpu_fused.cpp @@ -15,6 +15,10 @@ namespace { constexpr int NL = 35, HID = 1536, NQH = 8, PLE_DIM = 256; constexpr float RMS_EPS = 1e-6f, GATE_SCALE = 0.015625f, SOFTCAP = 30.0f; +// Fuse per-head rmsnorm+rope for q/k and fold the per-layer residual scalar into the post-PLE norm+add. +// Cuts non-gemv decode GPU work ~3% (Amdahl-capped to ~1% of total since gemv dominates). Default on. +static const bool g_fuse_nr = (std::getenv("CACTUS_NO_FUSE_NR") == nullptr); + struct LayerW { CactusQuantMatrix q, o, ple_down, ple_up, gate, up, down; CactusQuantMatrix k, v; @@ -210,20 +214,33 @@ bool CactusGraph::execute_gpu_fused() { size_t mx = km[1], ng = (hd+31)/32; void* xn = fresh(HID); cactus_metal_encode_rms_norm(xn, h, w.in_norm, 1, HID, RMS_EPS); - void* q = fresh(QD); cactus_metal_encode_quant_matmul(q, xn, &w.q); - void* qn = fresh(QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, NQH, hd, RMS_EPS); - void* qr = fresh(QD); cactus_metal_encode_rope(qr, qn, rc, rs, NQH, hd); + // q/k/v share input xn — batch their CQ4 transforms into one dispatch (non-shared layers). + void *cq=nullptr,*ck=nullptr,*cv=nullptr; bool qkv_batched=false; + if (!w.shared) { + cq=fresh(HID); ck=fresh(HID); cv=fresh(HID); + const CactusQuantMatrix* Wqkv[3]={&w.q,&w.k,&w.v}; void* codes[3]={cq,ck,cv}; + qkv_batched = cactus_metal_encode_transform_batch(xn, Wqkv, 3, codes); + } + void* q = fresh(QD); if(qkv_batched) cactus_metal_encode_gemv_precoded(q, cq, &w.q); else cactus_metal_encode_quant_matmul(q, xn, &w.q); + void* qr = fresh(QD); + if (!g_fuse_nr || !cactus_metal_encode_rms_norm_rope(qr, q, w.q_norm, rc, rs, NQH, hd, RMS_EPS)) { + void* qn = fresh(QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, NQH, hd, RMS_EPS); + cactus_metal_encode_rope(qr, qn, rc, rs, NQH, hd); + } size_t hist, total; const void *knewp, *vnewp; size_t kv_start = 0, kv_end; const uint32_t Wn = w.global ? 0u : (uint32_t)(km[1] - km[4] - 1); const uint32_t Sn = w.global ? 0u : (uint32_t)km[4]; const uint32_t Rn = (Wn > Sn) ? (Wn - Sn) : 1u; if (!w.shared) { - void* k = fresh(hd); cactus_metal_encode_quant_matmul(k, xn, &w.k); - void* v = fresh(hd); cactus_metal_encode_quant_matmul(v, xn, &w.v); - void* kn = fresh(hd); cactus_metal_encode_rms_norm(kn, k, w.k_norm, 1, hd, RMS_EPS); + void* k = fresh(hd); if(qkv_batched) cactus_metal_encode_gemv_precoded(k, ck, &w.k); else cactus_metal_encode_quant_matmul(k, xn, &w.k); + void* v = fresh(hd); if(qkv_batched) cactus_metal_encode_gemv_precoded(v, cv, &w.v); else cactus_metal_encode_quant_matmul(v, xn, &w.v); + void* kr = fresh(hd); + if (!g_fuse_nr || !cactus_metal_encode_rms_norm_rope(kr, k, w.k_norm, rc, rs, 1, hd, RMS_EPS)) { + void* kn = fresh(hd); cactus_metal_encode_rms_norm(kn, k, w.k_norm, 1, hd, RMS_EPS); + cactus_metal_encode_rope(kr, kn, rc, rs, 1, hd); + } void* vn = fresh(hd); cactus_metal_encode_rms_norm(vn, v, w.global ? G.vnorm_g : G.vnorm, 1, hd, RMS_EPS); - void* kr = fresh(hd); cactus_metal_encode_rope(kr, kn, rc, rs, 1, hd); size_t clen = km[0]; size_t slot = (!w.global && clen >= (size_t)Wn) ? (size_t)(Sn + ((clen - Sn) % Rn)) : clen; cactus_metal_encode_kv_append_i8(kr, (char*)w.kc+64, (char*)w.kc+64+mx*hd, @@ -256,8 +273,11 @@ bool CactusGraph::execute_gpu_fused() { const int M = w.mlp; void* xn2 = fresh(HID); cactus_metal_encode_rms_norm(xn2, h, w.pre_ffn, 1, HID, RMS_EPS); - void* gate= fresh(M); cactus_metal_encode_quant_matmul(gate, xn2, &w.gate); - void* up = fresh(M); cactus_metal_encode_quant_matmul(up, xn2, &w.up); + // gate/up share input xn2 — batch their transforms. + void *cg=fresh(HID), *cu=fresh(HID); const CactusQuantMatrix* Wgu[2]={&w.gate,&w.up}; void* cgu[2]={cg,cu}; + bool gu_batched = cactus_metal_encode_transform_batch(xn2, Wgu, 2, cgu); + void* gate= fresh(M); if(gu_batched) cactus_metal_encode_gemv_precoded(gate, cg, &w.gate); else cactus_metal_encode_quant_matmul(gate, xn2, &w.gate); + void* up = fresh(M); if(gu_batched) cactus_metal_encode_gemv_precoded(up, cu, &w.up); else cactus_metal_encode_quant_matmul(up, xn2, &w.up); void* g3 = fresh(M); cactus_metal_encode_swiglu(g3, gate, up, M, GATE_SCALE); void* mo = fresh(HID); cactus_metal_encode_quant_matmul(mo, g3, &w.down); void* h2 = fresh(HID); cactus_metal_encode_rms_norm_add(h2, mo, w.post_ffn, h, 1, HID, RMS_EPS); h = h2; @@ -265,10 +285,15 @@ bool CactusGraph::execute_gpu_fused() { void* ps = fresh(PLE_DIM); cactus_metal_encode_quant_matmul(ps, h, &w.ple_down); void* pm = fresh(PLE_DIM); cactus_metal_encode_swiglu(pm, ps, (const char*)pleBase + (size_t)l*PLE_DIM*2, PLE_DIM, 1.0f); void* pu = fresh(HID); cactus_metal_encode_quant_matmul(pu, pm, &w.ple_up); - void* h3 = fresh(HID); cactus_metal_encode_rms_norm_add(h3, pu, w.post_ple, h, 1, HID, RMS_EPS); h = h3; - + // fold the per-layer residual scalar into the post-PLE norm+residual (one pass instead of two) float ls = (float)(*(const __fp16*)w.scalar); - void* h4 = fresh(HID); cactus_metal_encode_scalar(2, h4, h, HID, ls); h = h4; + void* h3 = fresh(HID); + if (g_fuse_nr && cactus_metal_encode_rms_norm_add_scale(h3, pu, w.post_ple, h, 1, HID, RMS_EPS, ls)) { + h = h3; + } else { + cactus_metal_encode_rms_norm_add(h3, pu, w.post_ple, h, 1, HID, RMS_EPS); + void* h4 = fresh(HID); cactus_metal_encode_scalar(2, h4, h3, HID, ls); h = h4; + } } size_t V = B(3860).total_size; diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 10ab0e5f5..47d81ba2c 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -45,12 +45,19 @@ bool cactus_metal_encode_rope_m(void* out, const void* x, const void* cos_m, con uint32_t heads, uint32_t head_dim, uint32_t M); bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps); +bool cactus_metal_encode_rms_norm_rope(void* out, const void* x, const void* weight, + const void* cos, const void* sin, + uint32_t heads, uint32_t head_dim, float eps); bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* weight, const void* res, size_t rows, size_t dim, float eps); +bool cactus_metal_encode_rms_norm_add_scale(void* out, const void* in, const void* weight, const void* res, + size_t rows, size_t dim, float eps, float out_scale); bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3); bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_prec, size_t n); bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W); bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M); +bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* const* Ws, int B, void* const* codes); +bool cactus_metal_encode_gemv_precoded(void* out, const void* code, const CactusQuantMatrix* W); bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W); bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* code, const CactusQuantMatrix* W); diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index e310f4bbe..c8edefaac 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -62,6 +62,48 @@ kernel void cq4_transform_simd( code[b+2]=(half)zmem[perm[k+2]]; code[b+3]=(half)zmem[perm[k+3]]; } +// Batched CQ4 transform: one dispatch transforms the SAME input `x` for up to 3 weights that share the +// input (q/k/v use xn; gate/up use xn2 — verified `recip` is identical, only lsign/rsign/perm differ per +// weight). Grid = ng*B threadgroups; threadgroup (g + ng*b) transforms group g with weight b's params into +// code[b]. Replaces B separate transform dispatches + removes the g_code_buf WAR serialization. +kernel void cq4_transform_batch( + device const half* x [[buffer(0)]], + device const half* recip [[buffer(1)]], + device const char* lsign0 [[buffer(2)]], device const char* lsign1 [[buffer(3)]], device const char* lsign2 [[buffer(4)]], + device const char* rsign0 [[buffer(5)]], device const char* rsign1 [[buffer(6)]], device const char* rsign2 [[buffer(7)]], + device const uint* perm0 [[buffer(8)]], device const uint* perm1 [[buffer(9)]], device const uint* perm2 [[buffer(10)]], + device half* code0 [[buffer(11)]], device half* code1 [[buffer(12)]], device half* code2 [[buffer(13)]], + constant uint& ng [[buffer(14)]], + uint tgpos [[threadgroup_position_in_grid]], uint lane [[thread_index_in_simdgroup]], + threadgroup float* zmem [[threadgroup(0)]]) +{ + uint g = tgpos % ng, b = tgpos / ng; + device const char* lsign = (b==0u)?lsign0:((b==1u)?lsign1:lsign2); + device const char* rsign = (b==0u)?rsign0:((b==1u)?rsign1:rsign2); + device const uint* perm = (b==0u)?perm0 :((b==1u)?perm1 :perm2 ); + device half* code = (b==0u)?code0 :((b==1u)?code1 :code2 ); + uint base = g*128u + lane*4u; + uint k = lane*4u; + float x0=(float)x[base+0]*(float)recip[base+0]*(float)lsign[k+0]; + float x1=(float)x[base+1]*(float)recip[base+1]*(float)lsign[k+1]; + float x2=(float)x[base+2]*(float)recip[base+2]*(float)lsign[k+2]; + float x3=(float)x[base+3]*(float)recip[base+3]*(float)lsign[k+3]; + float a0=x0+x1,a1=x0-x1,a2=x2+x3,a3=x2-x3; + x0=a0+a2; x1=a1+a3; x2=a0-a2; x3=a1-a3; + #pragma clang loop unroll(full) + for (uint d=1u; d<=16u; d<<=1){ + bool hi=(lane&d)!=0u; + float p0=simd_shuffle_xor(x0,d),p1=simd_shuffle_xor(x1,d),p2=simd_shuffle_xor(x2,d),p3=simd_shuffle_xor(x3,d); + x0=hi?p0-x0:x0+p0; x1=hi?p1-x1:x1+p1; x2=hi?p2-x2:x2+p2; x3=hi?p3-x3:x3+p3; + } + float s=rsqrt(128.0f); + zmem[k+0]=x0*s*(float)rsign[k+0]; zmem[k+1]=x1*s*(float)rsign[k+1]; + zmem[k+2]=x2*s*(float)rsign[k+2]; zmem[k+3]=x3*s*(float)rsign[k+3]; + threadgroup_barrier(mem_flags::mem_threadgroup); + code[base+0]=(half)zmem[perm[k+0]]; code[base+1]=(half)zmem[perm[k+1]]; + code[base+2]=(half)zmem[perm[k+2]]; code[base+3]=(half)zmem[perm[k+3]]; +} + #define CQ4_VPL 16u kernel void cq4_gemv( device const half* code [[buffer(0)]], @@ -106,6 +148,58 @@ kernel void cq4_gemv( if (lane==0) y[n]=(half)acc; } +// Multi-row CQ4 gemv (M=1 decode): each simdgroup computes CQ4_NR output rows instead of 1. The shared +// activation `code` is read once per K-step and reused across the NR rows, while the NR weight streams are +// read interleaved (unrolled) — more independent memory requests in flight to hide M=1 latency (llama +// mul_mv `nr0` technique). ROWS*CQ4_NR rows per threadgroup; N is a multiple of 32 for all matmuls so no tail. +#define CQ4_NR 2u +kernel void cq4_gemv_mr( + device const half* code [[buffer(0)]], + device const uchar* packed [[buffer(1)]], + device const half* codebook [[buffer(2)]], + device const half* norms [[buffer(3)]], + device half* y [[buffer(4)]], + constant uint& gs [[buffer(5)]], + constant uint& num_groups [[buffer(6)]], + constant uint& pgb [[buffer(7)]], + constant uint& N [[buffer(8)]], + uint tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]]) +{ + threadgroup float cb[16]; + if (tl<16) cb[tl]=(float)codebook[tl]; + threadgroup_barrier(mem_flags::mem_threadgroup); + uint n0 = (tg*ROWS + sgid)*CQ4_NR; + if (n0>=N) return; + uint K = num_groups*gs; + float acc[CQ4_NR]; + #pragma clang loop unroll(full) + for (uint r=0;r>4)&0xF] + (float)c0.z*cb[(w[0]>>8)&0xF] + (float)c0.w*cb[(w[0]>>12)&0xF] + + (float)c1.x*cb[w[1]&0xF] + (float)c1.y*cb[(w[1]>>4)&0xF] + (float)c1.z*cb[(w[1]>>8)&0xF] + (float)c1.w*cb[(w[1]>>12)&0xF] + + (float)c2.x*cb[w[2]&0xF] + (float)c2.y*cb[(w[2]>>4)&0xF] + (float)c2.z*cb[(w[2]>>8)&0xF] + (float)c2.w*cb[(w[2]>>12)&0xF] + + (float)c3.x*cb[w[3]&0xF] + (float)c3.y*cb[(w[3]>>4)&0xF] + (float)c3.z*cb[(w[3]>>8)&0xF] + (float)c3.w*cb[(w[3]>>12)&0xF]; + acc[r] += (float)norms[(size_t)n*num_groups+g]*p; + } + } + #pragma clang loop unroll(full) + for (uint r=0;r= N*num_groups) return; + uint n = gid / num_groups, g = gid % num_groups; + uint K = num_groups*gs; + float nrm = (float)norms[(size_t)n*num_groups + g]; + device const uchar* pr = packed + ((size_t)n*num_groups + g)*pgb; + device half* wr = wf16 + (size_t)n*K + (size_t)g*gs; + for (uint e=0; e>1]; + uint nib = (e&1u)?(uint)(by>>4):(uint)(by&0xF); + wr[e] = (half)(nrm * (float)codebook[nib]); + } +} + +// Dense fp16 SG_MAT gemm: out[M,N] = code[M,K] @ dequant(W)[N,K]^T, reading pre-dequantized wf16 (no +// codebook LUT in the fill). Same tile/schedule as cq4_gemm_mma (64N x 32M x 32K, 128 threads) but the +// weight fill is a direct load, so it hits the dense microbench rate (~5257 vs ~3522 GFLOP/s dequant-in-fill). +kernel void cq4_gemm_dense_f16( + device const half* code [[buffer(0)]], + device const half* wf16 [[buffer(1)]], + device half* y [[buffer(2)]], + constant uint& K [[buffer(3)]], constant uint& N [[buffer(4)]], constant uint& M [[buffer(5)]], + uint2 tg [[threadgroup_position_in_grid]], uint tl [[thread_index_in_threadgroup]], uint sg [[simdgroup_index_in_threadgroup]]) +{ + threadgroup float pool[2048]; + threadgroup half* sa = (threadgroup half*)pool; + threadgroup half* sb = sa + 2048u; + threadgroup float* Cs = pool; + const uint NK=32u, NL0=2u, NL1=4u; + uint r0 = tg.y*64u, r1 = tg.x*32u; + uint lr0 = tl/NL0, lr1 = tl/NL1, il0 = tl%NL0; + uint nrow = r0+lr0; + uint iy = 8u*(tl%NL1); + device const half* yrow = code + (size_t)(r1+lr1)*K; + device const half* wrow = wf16 + (size_t)nrow*K; + simdgroup_matrix ma[4], mb[2]; + simdgroup_matrix mc[8]; + for (uint i=0;i<8u;i++) mc[i]=make_filled_simdgroup_matrix(0.f); + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint loop_k=0; loop_k>1u), n_sg = 32u*(sg&1u); + for (uint i=0;i<8u;i++) + simdgroup_store(mc[i], &Cs[(m_sg+8u*(i/4u))*64u + n_sg+8u*(i%4u)], 64u); + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint idx=tl; idx<32u*64u; idx+=128u){ + uint m=idx/64u, n=idx%64u; + if (r1+m0; s>>=1){ if (t0; s>>=1){ if (t #include #include +#include #include "cactus_kernels_msl.h" @@ -26,31 +27,36 @@ id queue = nil; id psoT = nil; id psoTsimd = nil; - id psoG = nil; + id psoTbatch = nil; + id psoG = nil, psoGmr = nil; id psoTm = nil, psoGm = nil; - id psoGmma = nil; + id psoGmma = nil, psoGdense = nil, psoDeq = nil; id psoRotate = nil; id psoEmbO = nil, psoEmbH = nil; id psoEmbOm = nil, psoEmbHm = nil; - id psoCopy=nil, psoBinary=nil, psoScalar=nil, psoUnary=nil, psoRms=nil, psoSwiglu=nil, psoRmsAdd=nil; + id psoCopy=nil, psoBinary=nil, psoScalar=nil, psoUnary=nil, psoRms=nil, psoSwiglu=nil, psoRmsAdd=nil, psoRmsAddScale=nil; id psoCF16F32=nil, psoCF32F16=nil, psoCI8F16=nil, psoCF16I8=nil; id psoAttn=nil, psoAttnC=nil, psoStrided=nil, psoScatter=nil, psoBcast=nil, psoKvAppend=nil; id psoAttnPre=nil, psoAttnPreMma2=nil, psoAttnPreHd256=nil; id psoKvAppendM=nil, psoKvAppendRingM=nil; id psoSlideS=nil, psoSlideR=nil, psoSlideRM=nil; - id psoRope=nil, psoRopeM=nil; + id psoRope=nil, psoRopeM=nil, psoRmsRope=nil; id psoArgmax=nil; id psoGather=nil; id dummy=nil; bool ok = false; MetalCtx() { @autoreleasepool { + const bool prof = getenv("CACTUS_PROF_INIT")!=nullptr; + auto t0 = std::chrono::high_resolution_clock::now(); + auto ms = [&](){ auto n=std::chrono::high_resolution_clock::now(); return std::chrono::duration(n-t0).count(); }; dev = MTLCreateSystemDefaultDevice(); if (!dev) return; queue = [dev newCommandQueue]; NSError* err = nil; id lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kCactusMSL] options:nil error:&err]; + if (prof) fprintf(stderr,"[prof-init] MSL compile: %.1fms\n", ms()); if (!lib) { if (err) fprintf(stderr,"[cactus-metal] MSL compile failed: %s\n",[[err localizedDescription] UTF8String]); return; } auto pso = [&](const char* name) -> id { @@ -60,14 +66,15 @@ if (!p) fprintf(stderr,"[cactus-metal] pipeline '%s' failed: %s\n", name, e?[[e localizedDescription] UTF8String]:"function not found"); return p; }; - psoT=pso("cq4_transform"); psoTsimd=pso("cq4_transform_simd"); psoG=pso("cq4_gemv"); + psoT=pso("cq4_transform"); psoTsimd=pso("cq4_transform_simd"); psoG=pso("cq4_gemv"); psoTbatch=pso("cq4_transform_batch"); psoGmr=pso("cq4_gemv_mr"); psoTm=pso("cq4_transform_m"); psoGm=pso("cq4_gemm"); psoGmma=pso("cq4_gemm_mma"); + psoGdense=pso("cq4_gemm_dense_f16"); psoDeq=pso("cq4_dequant_f16"); psoRotate=pso("lmhead_rotate"); psoEmbO=pso("emb_ortho"); psoEmbH=pso("emb_hadamard"); psoEmbOm=pso("emb_ortho_m"); psoEmbHm=pso("emb_hadamard_m"); psoGather=pso("gather_f16"); psoCopy=pso("copy_bytes"); psoBinary=pso("binary_f16"); psoScalar=pso("scalar_f16"); - psoUnary=pso("unary_f16"); psoRms=pso("rms_norm_f16"); psoSwiglu=pso("swiglu_f16"); psoRmsAdd=pso("rms_norm_add_f16"); + psoUnary=pso("unary_f16"); psoRms=pso("rms_norm_f16"); psoSwiglu=pso("swiglu_f16"); psoRmsAdd=pso("rms_norm_add_f16"); psoRmsAddScale=pso("rms_norm_add_scale_f16"); psoCF16F32=pso("cast_f16_f32"); psoCF32F16=pso("cast_f32_f16"); psoCI8F16=pso("cast_i8_f16"); psoCF16I8=pso("cast_f16_i8"); psoAttn=pso("attn_decode_i8"); psoAttnC=pso("attn_decode_combine"); @@ -75,8 +82,9 @@ psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoAttnPreHd256=pso("attn_prefill_mma_hd256"); psoKvAppendM=pso("kv_append_i8_m"); psoKvAppendRingM=pso("kv_append_ring_i8_m"); psoSlideS=pso("kv_slide_save"); psoSlideR=pso("kv_slide_restore"); psoSlideRM=pso("kv_slide_restore_m"); - psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); psoRopeM=pso("rope_f16_m"); + psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); psoRopeM=pso("rope_f16_m"); psoRmsRope=pso("rms_norm_rope_f16"); psoArgmax=pso("argmax_logits"); + if (prof) fprintf(stderr,"[prof-init] pso creation: %.1fms total\n", ms()); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoRopeM&&psoArgmax&&psoGather; @@ -93,6 +101,7 @@ struct ResW { id packed=nil, norms=nil, codebook=nil, lsign=nil, rsign=nil, perm=nil, recip=nil; id rotation=nil; + id wf16=nil; // lazily-dequantized dense fp16 weight (prefill pre-dequant path) }; std::unordered_map g_resident; std::mutex g_resident_mu; @@ -166,7 +175,11 @@ void deinterleave_4row(const CactusQuantMatrix* W, // Per-op-type GPU-time profiler: flush+time each dispatch, attribute to the current op tag g_op. // Perturbing (serializes), but gives an exact per-category GPU-time breakdown of a forward pass. static const bool g_prof_ops = (std::getenv("CACTUS_PROF_OPS") != nullptr); +static const bool g_gemv_mr = (std::getenv("CACTUS_NO_GEMV_MR") == nullptr); // multi-row M=1 gemv (default on; CACTUS_NO_GEMV_MR disables) +static const uint32_t g_mr_rows = 8u * 2u; // ROWS * CQ4_NR — MUST match cactus_kernels.metal CQ4_NR static const bool g_skip_mm=(std::getenv("CACTUS_SKIP_MM")!=nullptr), g_skip_attn=(std::getenv("CACTUS_SKIP_ATTN")!=nullptr), g_skip_norm=(std::getenv("CACTUS_SKIP_NORM")!=nullptr); +static const bool g_skip_transform=(std::getenv("CACTUS_SKIP_TRANSFORM")!=nullptr); +static const bool g_predequant=(std::getenv("CACTUS_NO_PREDEQUANT")==nullptr); // prefill: dense fp16 gemm on pre-dequantized weights (default on; +~1.25GB RAM) static const char* g_op = "other"; struct OpProf { std::map> t; @@ -362,6 +375,23 @@ bool cactus_metal_encode_rope_m(void* out, const void* x, const void* cos_m, con barrier(); return true; } +bool cactus_metal_encode_rms_norm_rope(void* out, const void* x, const void* weight, + const void* cos, const void* sin, + uint32_t heads, uint32_t head_dim, float eps) { + if (!ctx().ok || !ctx().psoRmsRope) return false; + ensureEncoder(); + g_op="rmsnorm"; + uint32_t total=heads*head_dim; uint32_t hd=head_dim; float e=eps; + [g_enc setComputePipelineState:ctx().psoRmsRope]; + setBufAt(x, (size_t)total*2, 0); setBufAt(weight, (size_t)head_dim*2, 1); + setBufAt(cos, (size_t)head_dim*2, 2); setBufAt(sin, (size_t)head_dim*2, 3); + setBufAt(out, (size_t)total*2, 4); + [g_enc setBytes:&hd length:4 atIndex:5]; [g_enc setBytes:&e length:4 atIndex:6]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + if(!g_skip_norm) [g_enc dispatchThreadgroups:MTLSizeMake(heads,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps) { if (!ctx().ok) return false; @@ -390,6 +420,20 @@ bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* wei barrier(); return true; } +bool cactus_metal_encode_rms_norm_add_scale(void* out, const void* in, const void* weight, const void* res, + size_t rows, size_t dim, float eps, float out_scale) { + if (!ctx().ok || !ctx().psoRmsAddScale) return false; + ensureEncoder(); + g_op="rmsnorm"; + uint32_t d=(uint32_t)dim; float e=eps, os=out_scale; + [g_enc setComputePipelineState:ctx().psoRmsAddScale]; + setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(res, rows*dim*2, 2); setBufAt(out, rows*dim*2, 3); + [g_enc setBytes:&d length:4 atIndex:4]; [g_enc setBytes:&e length:4 atIndex:5]; [g_enc setBytes:&os length:4 atIndex:6]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + if(!g_skip_norm) [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) { if (!ctx().ok) return false; ensureEncoder(); @@ -456,7 +500,8 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; barrier(); - [g_enc setComputePipelineState:ctx().psoG]; + bool umr = g_gemv_mr && ctx().psoGmr && (N % g_mr_rows == 0u); + [g_enc setComputePipelineState:(umr?ctx().psoGmr:ctx().psoG)]; [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:0 atIndex:1]; [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; setBufAt(out, (size_t)N*2, 4); @@ -464,7 +509,57 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; uint32_t ROWS=8; g_op="matmul"; - if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + uint32_t grid = umr ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; + if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + barrier(); + return true; +} + +// Batched CQ4 transform for up to 3 weights sharing the input `x` (recip). One dispatch instead of B. +bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* const* Ws, int B, void* const* codes) { + if (!ctx().ok || B < 1 || B > 3 || !ctx().psoTbatch) return false; + const uint32_t gs=Ws[0]->group_size, K=Ws[0]->K, ng=Ws[0]->num_groups; + if (gs != 128u || !quant_fast_eligible(Ws[0])) return false; + ResW& r0 = resident(Ws[0]); + ResW* rw[3] = { &r0, &r0, &r0 }; + for (int b=1;blsign offset:0 atIndex:2+b]; + [g_enc setBuffer:R->rsign offset:0 atIndex:5+b]; + [g_enc setBuffer:R->perm offset:0 atIndex:8+b]; + setBufAt(codes[bgroup_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; + if (!quant_fast_eligible(W)) return false; + ResW& rw = resident(W); + ensureEncoder(); + g_op = "matmul"; + bool umr = g_gemv_mr && ctx().psoGmr && (N % g_mr_rows == 0u); + [g_enc setComputePipelineState:(umr?ctx().psoGmr:ctx().psoG)]; + setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:0 atIndex:1]; + [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; + setBufAt(out, (size_t)N*2, 4); + [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; + [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; + uint32_t ROWS=8; + uint32_t grid = umr ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; + if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; barrier(); return true; } @@ -495,9 +590,33 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setBytes:&K length:4 atIndex:7]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; g_op="transform"; - [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; + if (!g_skip_transform) [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; barrier(); + // Pre-dequant dense path: dequantize the weight to fp16 ONCE (cached in rw.wf16), then run a plain + // fp16 SG_MAT gemm — avoids the codebook LUT in the gemm fill (~33% faster gemm). The one-time dequant + // is triggered inside the load-time prefill warmup, so real prefill only pays the fast dense gemm. + if (g_predequant && ctx().psoGdense && ctx().psoDeq) { + if (!rw.wf16) { + rw.wf16 = [ctx().dev newBufferWithLength:(size_t)N*K*2 options:MTLResourceStorageModeShared]; + [g_enc setComputePipelineState:ctx().psoDeq]; + [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; + [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.wf16 offset:0 atIndex:3]; + [g_enc setBytes:&gs length:4 atIndex:4]; [g_enc setBytes:&ng length:4 atIndex:5]; + [g_enc setBytes:&pgb length:4 atIndex:6]; [g_enc setBytes:&N length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake((size_t)N*ng,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + } + [g_enc setComputePipelineState:ctx().psoGdense]; + [g_enc setBuffer:g_code_buf_m offset:0 atIndex:0]; [g_enc setBuffer:rw.wf16 offset:0 atIndex:1]; + setBufAt(out, (size_t)M*N*2, 2); + [g_enc setBytes:&K length:4 atIndex:3]; [g_enc setBytes:&N length:4 atIndex:4]; [g_enc setBytes:&M length:4 atIndex:5]; + g_op="matmul"; + if (!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + barrier(); + return true; + } + bool mma = (ctx().psoGmma != nil); [g_enc setComputePipelineState:(mma?ctx().psoGmma:ctx().psoGm)]; [g_enc setBuffer:g_code_buf_m offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:0 atIndex:1]; @@ -532,14 +651,16 @@ bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* co [g_enc dispatchThreads:MTLSizeMake(K,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; barrier(); - [g_enc setComputePipelineState:ctx().psoG]; + bool umr = g_gemv_mr && ctx().psoGmr && (N % g_mr_rows == 0u); + [g_enc setComputePipelineState:(umr?ctx().psoGmr:ctx().psoG)]; setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:0 atIndex:1]; [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; setBufAt(out, (size_t)N*2, 4); [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; uint32_t ROWS=8; - if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + uint32_t grid = umr ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; + if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; barrier(); return true; } @@ -751,7 +872,7 @@ bool cactus_metal_encode_attention_i8_prefill( const uint32_t T = 128; if (total_keys == 0 || M == 0 || (num_q_heads % num_kv_heads) != 0) return false; uint32_t maxsc = (ring > 0u) ? ((total_keys > sink + ring) ? (sink + ring) : total_keys) : 256u; - static const bool en_hd256 = (std::getenv("CACTUS_HD256") != nullptr); + static const bool en_hd256 = (std::getenv("CACTUS_NO_HD256") == nullptr); // tiled local (hd=256) prefill attention (default on) bool mma2 = (ring == 0u && is_causal && total_keys >= 2048u && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u); // Route ALL global (hd=512, full-attention / window==0) layers through the tiled mma2 kernel, not just // the long-context ones — the tiled kernel loads each KV tile into shared once and reuses it across a diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 9c7ce4407..24132cf21 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -24,11 +24,15 @@ bool cactus_metal_encode_swiglu(void*, const void*, const void*, size_t, float) bool cactus_metal_encode_rope(void*, const void*, const void*, const void*, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_rope_m(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_rms_norm(void*, const void*, const void*, size_t, size_t, float) { return false; } +bool cactus_metal_encode_rms_norm_rope(void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, float) { return false; } bool cactus_metal_encode_rms_norm_add(void*, const void*, const void*, const void*, size_t, size_t, float) { return false; } +bool cactus_metal_encode_rms_norm_add_scale(void*, const void*, const void*, const void*, size_t, size_t, float, float) { return false; } bool cactus_metal_encode_argmax(const void*, uint32_t, void*) { return false; } bool cactus_metal_encode_cast(void*, int, const void*, int, size_t) { return false; } bool cactus_metal_encode_quant_matmul(void*, const void*, const CactusQuantMatrix*) { return false; } bool cactus_metal_encode_quant_matmul_m(void*, const void*, const CactusQuantMatrix*, uint32_t) { return false; } +bool cactus_metal_encode_transform_batch(const void*, const CactusQuantMatrix* const*, int, void* const*) { return false; } +bool cactus_metal_encode_gemv_precoded(void*, const void*, const CactusQuantMatrix*) { return false; } bool cactus_metal_prewarm_quant(const CactusQuantMatrix*) { return false; } bool cactus_metal_encode_quant_matmul_ortho(void*, const void*, void*, const CactusQuantMatrix*) { return false; } bool cactus_metal_encode_embedding_ortho(void*, uint32_t, const CactusQuantMatrix*) { return false; } diff --git a/python/cactus/cli/runtime.py b/python/cactus/cli/runtime.py index 9e162cddb..f9e051865 100644 --- a/python/cactus/cli/runtime.py +++ b/python/cactus/cli/runtime.py @@ -77,6 +77,7 @@ def _link_python_runtime_library(*, static_lib, library_path): "-framework", "Accelerate", "-framework", "CoreML", "-framework", "Foundation", + "-framework", "Metal", "-framework", "Security", "-framework", "SystemConfiguration", "-framework", "CFNetwork", diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 32f322b2a..fb6f09988 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -223,16 +223,25 @@ def _tile_to_length(tensor: torch.Tensor, length: int) -> torch.Tensor: def _resolve_model_pad_token_id(model: torch.nn.Module) -> int | None: + def _coerce(value: object) -> int | None: + if isinstance(value, (list, tuple)): + value = value[0] if value else None + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None config = getattr(model, "config", None) for attr_name in ("pad_token_id", "eos_token_id", "bos_token_id"): - value = getattr(config, attr_name, None) - if value is not None: - return int(value) + resolved = _coerce(getattr(config, attr_name, None)) + if resolved is not None: + return resolved generation_config = getattr(model, "generation_config", None) for attr_name in ("pad_token_id", "eos_token_id", "bos_token_id"): - value = getattr(generation_config, attr_name, None) - if value is not None: - return int(value) + resolved = _coerce(getattr(generation_config, attr_name, None)) + if resolved is not None: + return resolved return None @@ -1162,6 +1171,7 @@ def _gemma4_text_attention_forward( hidden_states=hidden_states, position_embeddings=position_embeddings, attention_mask=attention_mask, + shared_kv_states={}, position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, @@ -4281,6 +4291,138 @@ def _build_gemma3_causal_lm_component_specs( return specs +def _build_gemma4_causal_lm_component_specs( + model: torch.nn.Module, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None = None, + components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, + dynamic_batch: bool = True, + max_slots: int = 1, +) -> list[ComponentModuleSpec] | None: + input_ids = named_tensors.get("input_ids") + if input_ids is None: + return None + pad_token_id = _resolve_model_pad_token_id(model) + requested_set = set(components or ()) + if not requested_set or "decoder" in requested_set: + requested_set |= { + "decoder", + "lm_encoder_step", + "lm_encoder_text_chunk", + "decoder_prefill_chunk", + "decoder_step", + } + common_graph_meta = { + "weights_dir": weights_dir, + "task": "causal_lm_logits", + "adapter_family": "gemma4", + } + metadata = {"family": "gemma4", "task": "causal_lm_logits"} + + specs: list[ComponentModuleSpec] = [] + if "decoder" in requested_set: + decoder = Gemma4CausalLMLogitsAdapter(model, pad_token_id=pad_token_id).eval() + specs.append(ComponentModuleSpec( + component="decoder", + module=decoder, + example_inputs=(input_ids,), + input_keys=("input_ids",), + output_keys=("logits",), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata=metadata, + )) + + cached_components = {"lm_encoder_step", "lm_encoder_text_chunk", "decoder_prefill_chunk", "decoder_step"} + if not (cached_components & requested_set): + return specs + + lm_encoder_step = Gemma4LMEncoderStepAdapter(model, weights_dir=weights_dir).eval() + lm_encoder_text_chunk = Gemma4LMEncoderTextChunkAdapter(model, weights_dir=weights_dir).eval() + decoder_prefill_chunk = Gemma4DecoderPrefillChunkAdapter(model, weights_dir=weights_dir).eval() + decoder_step = Gemma4DecoderStepAdapter(model, weights_dir=weights_dir).eval() + cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) + prefill_chunk_size = max(2, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "128") or "128")) + + step_input_ids = input_ids[:, -1:].contiguous() + step_position_ids = torch.full( + (int(input_ids.shape[0]), 1), + max(0, int(input_ids.shape[1]) - 1), + dtype=torch.long, + device=input_ids.device, + ) + chunk_input_ids = _tile_to_length(input_ids, prefill_chunk_size) + chunk_position_ids = torch.arange( + prefill_chunk_size, dtype=torch.long, device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + with torch.no_grad(): + decoder_step_inputs = lm_encoder_step(step_input_ids, step_position_ids) + prefill_decoder_inputs = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) + + if "lm_encoder_step" in requested_set: + specs.append(ComponentModuleSpec( + component="lm_encoder_step", + module=lm_encoder_step, + example_inputs=(step_input_ids, step_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, + metadata=metadata, + dynamic_batch_axis=0 if dynamic_batch else None, + )) + if "lm_encoder_text_chunk" in requested_set: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + graph_meta={ + **common_graph_meta, + "component": "lm_encoder_text_chunk", + "encoder_chunk_size": prefill_chunk_size, + }, + metadata=metadata, + )) + if "decoder_prefill_chunk" in requested_set: + specs.append(ComponentModuleSpec( + component="decoder_prefill_chunk", + module=decoder_prefill_chunk, + example_inputs=tuple(prefill_decoder_inputs), + input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_prefill_chunk", + "use_internal_kv_cache": True, + "max_cache_seq_len": cache_seq_len, + "cache_sink_size": 4, + "prefill_chunk_size": prefill_chunk_size, + }, + metadata=metadata, + )) + if "decoder_step" in requested_set: + specs.append(ComponentModuleSpec( + component="decoder_step", + module=decoder_step, + example_inputs=tuple(decoder_step_inputs), + input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + output_keys=("logits", "probe_hidden"), + graph_meta={ + **common_graph_meta, + "component": "decoder_step", + "use_internal_kv_cache": True, + "max_cache_seq_len": cache_seq_len, + "cache_sink_size": 4, + "cache_num_slots": (max_slots if dynamic_batch else 1), + }, + metadata=metadata, + dynamic_batch_axis=0 if dynamic_batch else None, + )) + return specs + + def _gemma4_make_audio_mask_fp16_safe(model: torch.nn.Module) -> None: backbone = getattr(model, "model", model) audio_tower = getattr(backbone, "audio_tower", None) @@ -6978,6 +7120,16 @@ def build_component_module_specs( dynamic_batch=dynamic_batch, max_slots=max_slots, ) + if family == "gemma4" and task == "causal_lm_logits": + return _build_gemma4_causal_lm_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + components=components, + cache_context_length=cache_context_length, + dynamic_batch=dynamic_batch, + max_slots=max_slots, + ) if family == "gemma4" and task == "multimodal_causal_lm_logits": return _build_gemma4_multimodal_component_specs( model, From 49246f0d60f44b06433a7bdcdf5f30152866880a Mon Sep 17 00:00:00 2001 From: rshemet <42696921+rshemet@users.noreply.github.com> Date: Wed, 1 Jul 2026 07:57:32 -0700 Subject: [PATCH 07/36] Fix C++ test build: expose vendored libs include (picojson) (#748) gemma_tools.h (reached transitively by the test binaries via utils.h) includes picojson.h from libs/, but that dir was a PRIVATE include of cactus_engine, so it was not propagated to targets that link the engine. Make libs a PUBLIC include. Signed-off-by: rshemet Signed-off-by: jakmro --- cactus-engine/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index b1b2a81d3..58e87cb86 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -48,8 +48,8 @@ set_target_properties(cactus_engine PROPERTIES target_include_directories(cactus_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/libs PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src - ${CMAKE_CURRENT_SOURCE_DIR}/libs ) target_link_libraries(cactus_engine PUBLIC cactus_graph) From 648c28a70bed27c3bc389cffcfc4869c3c52ca61 Mon Sep 17 00:00:00 2001 From: rshemet <42696921+rshemet@users.noreply.github.com> Date: Wed, 1 Jul 2026 12:23:50 -0700 Subject: [PATCH 08/36] Trie-based grammar constraints for Gemma force_tools (#747) Reuse the Needle char-trie machinery for the Gemma tool-call path: fixes function-name prefix shadowing (e.g. send vs send_message) and enforces required arguments and enum values on a single forced call. Tokenizer-agnostic (validates decoded token strings); Needle path unchanged. Adds regression tests. Signed-off-by: rshemet Signed-off-by: jakmro --- cactus-engine/src/complete.cpp | 41 +- cactus-engine/src/constraints.cpp | 598 +++++++++++---------- cactus-engine/src/engine.h | 87 +-- python/tests/test_tool_constraint_state.py | 181 ++++++- 4 files changed, 564 insertions(+), 343 deletions(-) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 51ef39b06..85273c3c9 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -25,6 +25,8 @@ namespace { std::vector> extract_schema_property_types(const std::string& schema); std::vector extract_schema_required(const std::string& schema); +std::vector extract_string_array(const std::string& json, const std::string& key); +std::string extract_json_object_field(const std::string& json, const std::string& key); std::string extract_last_user_query(const std::vector& messages) { for (auto it = messages.rbegin(); it != messages.rend(); ++it) { @@ -65,9 +67,13 @@ std::vector build_tool_constraint_specs(const std::vectorsecond); + std::string properties_object = extract_json_object_field(schema_it->second, "properties"); spec.parameter_names.reserve(properties.size()); + spec.parameter_enums.reserve(properties.size()); for (const auto& [name, _] : properties) { spec.parameter_names.push_back(name); + spec.parameter_enums.push_back( + extract_string_array(extract_json_object_field(properties_object, name), "enum")); } spec.required_parameter_names = extract_schema_required(schema_it->second); } @@ -194,25 +200,30 @@ std::vector> extract_schema_property_types(c return properties; } -std::vector extract_schema_required(const std::string& schema) { - std::vector required; - std::string key = "\"required\""; - size_t key_pos = schema.find(key); - if (key_pos == std::string::npos) return required; - size_t arr_start = schema.find('[', key_pos + key.size()); - if (arr_start == std::string::npos) return required; - size_t arr_end = schema.find(']', arr_start); - if (arr_end == std::string::npos) return required; +std::vector extract_string_array(const std::string& json, const std::string& key) { + std::vector values; + if (json.empty()) return values; + std::string pattern = "\"" + key + "\""; + size_t key_pos = json.find(pattern); + if (key_pos == std::string::npos) return values; + size_t arr_start = json.find('[', key_pos + pattern.size()); + if (arr_start == std::string::npos) return values; + size_t arr_end = json.find(']', arr_start); + if (arr_end == std::string::npos) return values; size_t pos = arr_start + 1; while (pos < arr_end) { - size_t qs = schema.find('"', pos); + size_t qs = json.find('"', pos); if (qs == std::string::npos || qs >= arr_end) break; - size_t qe = schema.find('"', qs + 1); + size_t qe = json.find('"', qs + 1); if (qe == std::string::npos || qe > arr_end) break; - required.push_back(schema.substr(qs + 1, qe - qs - 1)); + values.push_back(json.substr(qs + 1, qe - qs - 1)); pos = qe + 1; } - return required; + return values; +} + +std::vector extract_schema_required(const std::string& schema) { + return extract_string_array(schema, "required"); } static bool json_object_has_key(const std::string& obj, const std::string& key) { @@ -1141,6 +1152,10 @@ int cactus_complete( : "kept local"; } + if (prompt.options.force_tools && !prompt.tools.empty() && primary_function_calls.empty()) { + CACTUS_LOG_WARN("force_tools", "force_tools was set but no tool call was parsed from the constrained output"); + } + std::string result = construct_response_json(primary_response, primary_function_calls, time_to_first_token, total_time_ms, prefill_tps, decode_tps, prompt_tokens, completion_tokens, confidence, handoff_succeeded, diff --git a/cactus-engine/src/constraints.cpp b/cactus-engine/src/constraints.cpp index 599d3e8ed..ec6731dbd 100644 --- a/cactus-engine/src/constraints.cpp +++ b/cactus-engine/src/constraints.cpp @@ -5,18 +5,14 @@ namespace engine { constexpr float FORCE_BIAS = 500.0f; constexpr float BLOCK_BIAS = -500.0f; -constexpr float NEEDLE_BLOCK_BIAS = -1e9f; - -static bool contains_odd_quote_tags(const std::string& text) { - static constexpr const char* kQuoteTag = "<|\"|>"; - static constexpr size_t kQuoteTagLen = 5; - bool odd = false; - size_t pos = 0; - while ((pos = text.find(kQuoteTag, pos)) != std::string::npos) { - odd = !odd; - pos += kQuoteTagLen; - } - return odd; +constexpr float TRIE_BLOCK_BIAS = -1e9f; + +static constexpr const char* kEscapeTag = "<|\"|>"; +static constexpr size_t kEscapeTagLen = 5; + +static bool ends_with_escape_tag(const std::string& text) { + return text.size() >= kEscapeTagLen && + text.compare(text.size() - kEscapeTagLen, kEscapeTagLen, kEscapeTag) == 0; } void ToolCallConstrainer::add_tokens_for_string(const std::string& str, std::unordered_set& token_set) { @@ -27,36 +23,6 @@ void ToolCallConstrainer::add_tokens_for_string(const std::string& str, std::uno } } -void ToolCallConstrainer::add_tokens_for_prefix_string(const std::string& prefix, std::unordered_set& token_set) { - if (!tokenizer_) return; - const uint32_t vocab = tokenizer_->get_vocab_size(); - for (uint32_t t = 0; t < vocab; t++) { - if (tokenizer_->decode({t}).rfind(prefix, 0) == 0) { - token_set.insert(t); - } - } -} - -void ToolCallConstrainer::tokenize_function_names(bool quote_names) { - all_func_name_tokens_.clear(); - func_name_sequences_.clear(); - - for (const auto& name : function_names_) { - std::string name_to_encode = quote_names ? ("\"" + name + "\"") : name; - auto tokens = tokenizer_->encode(name_to_encode); - func_name_sequences_[name] = tokens; - for (uint32_t t : tokens) { - all_func_name_tokens_.insert(t); - } - if (quote_names) { - auto unquoted_tokens = tokenizer_->encode(name); - for (uint32_t t : unquoted_tokens) { - all_func_name_tokens_.insert(t); - } - } - } -} - void ToolCallConstrainer::init_common_tokens() { backtick_tokens_.clear(); add_tokens_for_string("`", backtick_tokens_); @@ -70,22 +36,22 @@ void ToolCallConstrainer::init_common_tokens() { add_tokens_for_string("` ", backtick_tokens_); } -void ToolCallConstrainer::needle_insert_word(NeedleTrieNode* root, const std::string& word) { +void ToolCallConstrainer::trie_insert(TrieNode* root, const std::string& word) { if (!root) return; - NeedleTrieNode* node = root; + TrieNode* node = root; for (char ch : word) { auto& child = node->children[ch]; - if (!child) child = std::make_unique(); + if (!child) child = std::make_unique(); node = child.get(); } node->is_terminal = true; } -const ToolCallConstrainer::NeedleTrieNode* ToolCallConstrainer::needle_get_trie_node( - const NeedleTrieNode* root, +const ToolCallConstrainer::TrieNode* ToolCallConstrainer::trie_seek( + const TrieNode* root, const std::string& prefix ) const { - const NeedleTrieNode* node = root; + const TrieNode* node = root; for (char ch : prefix) { if (!node) return nullptr; auto it = node->children.find(ch); @@ -95,12 +61,12 @@ const ToolCallConstrainer::NeedleTrieNode* ToolCallConstrainer::needle_get_trie_ return node; } -bool ToolCallConstrainer::needle_check_token_valid(const std::string& token_text, - const NeedleTrieNode* trie_node) const { - const NeedleTrieNode* node = trie_node; +bool ToolCallConstrainer::trie_token_valid(const std::string& token_text, + const TrieNode* node, + char terminator) const { if (!node) return false; for (char ch : token_text) { - if (ch == '"') return node->is_terminal; + if (ch == terminator) return node->is_terminal; auto it = node->children.find(ch); if (it == node->children.end()) return false; node = it->second.get(); @@ -108,28 +74,165 @@ bool ToolCallConstrainer::needle_check_token_valid(const std::string& token_text return true; } -void ToolCallConstrainer::reset_needle_constraints() { - needle_json_state_ = NeedleJsonState::FREE; - needle_buffer_.clear(); - needle_constrained_buf_.clear(); - needle_current_function_.clear(); - needle_in_arguments_ = false; - needle_arguments_depth_ = 0; - needle_nesting_depth_ = 0; - needle_in_string_value_ = false; - needle_prev_char_escape_ = false; +const ToolCallConstrainer::TrieNode* ToolCallConstrainer::current_param_trie() const { + auto it = param_tries_.find(current_function_); + return it != param_tries_.end() ? it->second.get() : nullptr; +} + +void ToolCallConstrainer::mark_trie_bias(const TrieNode* node, char terminator, + const std::vector& extra_allowed, + int32_t terminal_token, bool exact_terminator) { + if (!node) return; + + std::vector valid(token_strings_.size(), false); + bool has_valid = false; + auto mark = [&](char first_char, bool unconditional) { + auto idx_it = token_index_.find(first_char); + if (idx_it == token_index_.end()) return; + for (uint32_t token_id : idx_it->second) { + if (valid[token_id]) continue; + if (unconditional || trie_token_valid(token_strings_[token_id], node, terminator)) { + valid[token_id] = true; + has_valid = true; + } + } + }; + + for (const auto& [first_char, _] : node->children) mark(first_char, false); + if (node->is_terminal) { + if (exact_terminator) { + auto idx_it = token_index_.find(terminator); + if (idx_it != token_index_.end()) { + for (uint32_t token_id : idx_it->second) { + if (!valid[token_id] && token_strings_[token_id].size() == 1) { + valid[token_id] = true; + has_valid = true; + } + } + } + } else { + mark(terminator, false); + } + } + for (char c : extra_allowed) mark(c, true); + if (node->is_terminal && terminal_token >= 0) { + uint32_t tid = static_cast(terminal_token); + if (tid < valid.size() && !valid[tid]) { + valid[tid] = true; + has_valid = true; + } + } + if (!has_valid) return; + + for (size_t token_id = 0; token_id < valid.size(); ++token_id) { + if (!valid[token_id]) { + current_bias_[static_cast(token_id)] = TRIE_BLOCK_BIAS; + } + } +} + +void ToolCallConstrainer::reset_constraint_state() { + region_ = Region::FREE; + constraint_buffer_.clear(); + constrained_buf_.clear(); + current_function_.clear(); + in_arguments_ = false; + arguments_depth_ = 0; + nesting_depth_ = 0; + in_string_value_ = false; + prev_char_escape_ = false; + at_key_start_ = false; + await_enum_open_ = false; + active_enum_trie_ = nullptr; + emitted_keys_.clear(); + remaining_key_trie_.reset(); +} + +void ToolCallConstrainer::rebuild_remaining_keys() { + remaining_key_trie_ = std::make_unique(); + for (const auto& tool : tool_specs_) { + if (tool.name != current_function_) continue; + for (const auto& param_name : tool.parameter_names) { + if (!param_name.empty() && !emitted_keys_.count(param_name)) { + trie_insert(remaining_key_trie_.get(), param_name); + } + } + break; + } +} + +bool ToolCallConstrainer::required_satisfied() const { + for (const auto& tool : tool_specs_) { + if (tool.name != current_function_) continue; + for (const auto& req : tool.required_parameter_names) { + if (!emitted_keys_.count(req)) return false; + } + return true; + } + return true; +} + +void ToolCallConstrainer::build_constraint_tables() { + name_trie_ = std::make_unique(); + param_tries_.clear(); + enum_tries_.clear(); + for (const auto& tool : tool_specs_) { + if (!tool.name.empty()) trie_insert(name_trie_.get(), tool.name); + + if (is_needle()) { + auto param_root = std::make_unique(); + for (const auto& param_name : tool.parameter_names) { + if (!param_name.empty()) trie_insert(param_root.get(), param_name); + } + param_tries_[tool.name] = std::move(param_root); + } + + for (size_t i = 0; i < tool.parameter_names.size() && i < tool.parameter_enums.size(); ++i) { + if (tool.parameter_enums[i].empty()) continue; + auto enum_root = std::make_unique(); + for (const auto& value : tool.parameter_enums[i]) { + if (!value.empty()) trie_insert(enum_root.get(), value); + } + enum_tries_[tool.name][tool.parameter_names[i]] = std::move(enum_root); + } + } + + const uint32_t vocab_size = tokenizer_ ? tokenizer_->get_vocab_size() : 0; + if (token_strings_.size() == vocab_size) return; + + token_strings_.assign(vocab_size, ""); + token_index_.clear(); + + const uint32_t eos = tokenizer_->get_eos_token(); + const uint32_t bos = tokenizer_->get_bos_token(); + const uint32_t unk = tokenizer_->get_unk_token(); + constexpr uint32_t pad = 0; + + for (uint32_t token_id = 0; token_id < vocab_size; ++token_id) { + if (token_id == pad || token_id == eos || token_id == bos || token_id == unk) continue; + std::string decoded = tokenizer_->decode({token_id}); + if (decoded.size() == 1) { + unsigned char b = static_cast(decoded[0]); + if (b >= 0x80) { + decoded.assign({static_cast(0xC0 | (b >> 6)), + static_cast(0x80 | (b & 0x3F))}); + } + } + token_strings_[token_id] = decoded; + if (!decoded.empty()) token_index_[decoded.front()].push_back(token_id); + } } bool ToolCallConstrainer::needle_at_arg_key_start() const { - if (needle_buffer_.size() < 2) return false; - return needle_buffer_.compare(needle_buffer_.size() - 2, 2, "{\"") == 0 || - needle_buffer_.compare(needle_buffer_.size() - 2, 2, ",\"") == 0; + if (constraint_buffer_.size() < 2) return false; + return constraint_buffer_.compare(constraint_buffer_.size() - 2, 2, "{\"") == 0 || + constraint_buffer_.compare(constraint_buffer_.size() - 2, 2, ",\"") == 0; } bool ToolCallConstrainer::needle_is_value_string_start() const { - if (needle_buffer_.empty()) return false; - for (size_t i = needle_buffer_.size() - 1; i-- > 0;) { - char ch = needle_buffer_[i]; + if (constraint_buffer_.empty()) return false; + for (size_t i = constraint_buffer_.size() - 1; i-- > 0;) { + char ch = constraint_buffer_[i]; if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r') continue; return ch == ':'; } @@ -137,69 +240,66 @@ bool ToolCallConstrainer::needle_is_value_string_start() const { } void ToolCallConstrainer::feed_needle_char(char ch) { - if (needle_json_state_ == NeedleJsonState::IN_NAME || - needle_json_state_ == NeedleJsonState::IN_ARG_KEY) { + if (region_ == Region::IN_NAME || region_ == Region::IN_ARG_KEY) { if (ch == '"') { - if (needle_json_state_ == NeedleJsonState::IN_NAME) { - needle_current_function_ = needle_constrained_buf_; - } - needle_constrained_buf_.clear(); - needle_json_state_ = NeedleJsonState::FREE; + if (region_ == Region::IN_NAME) current_function_ = constrained_buf_; + constrained_buf_.clear(); + region_ = Region::FREE; } else { - needle_constrained_buf_.push_back(ch); + constrained_buf_.push_back(ch); } - needle_buffer_.push_back(ch); + constraint_buffer_.push_back(ch); return; } - needle_buffer_.push_back(ch); + constraint_buffer_.push_back(ch); - if (needle_in_string_value_) { - if (needle_prev_char_escape_) { - needle_prev_char_escape_ = false; + if (in_string_value_) { + if (prev_char_escape_) { + prev_char_escape_ = false; return; } if (ch == '\\') { - needle_prev_char_escape_ = true; + prev_char_escape_ = true; return; } - if (ch == '"') needle_in_string_value_ = false; + if (ch == '"') in_string_value_ = false; return; } if (ch == '{' || ch == '[') { - needle_nesting_depth_++; + nesting_depth_++; } else if (ch == '}' || ch == ']') { - needle_nesting_depth_ = std::max(0, needle_nesting_depth_ - 1); - if (ch == '}' && needle_in_arguments_ && needle_nesting_depth_ < needle_arguments_depth_) { - needle_in_arguments_ = false; + nesting_depth_ = std::max(0, nesting_depth_ - 1); + if (ch == '}' && in_arguments_ && nesting_depth_ < arguments_depth_) { + in_arguments_ = false; } } - if (needle_buffer_.size() >= 8 && !needle_in_arguments_ && - needle_buffer_.compare(needle_buffer_.size() - 8, 8, "\"name\":\"") == 0) { - needle_json_state_ = NeedleJsonState::IN_NAME; - needle_constrained_buf_.clear(); + if (constraint_buffer_.size() >= 8 && !in_arguments_ && + constraint_buffer_.compare(constraint_buffer_.size() - 8, 8, "\"name\":\"") == 0) { + region_ = Region::IN_NAME; + constrained_buf_.clear(); return; } - if (needle_buffer_.size() >= 13 && - needle_buffer_.compare(needle_buffer_.size() - 13, 13, "\"arguments\":{") == 0) { - needle_in_arguments_ = true; - needle_arguments_depth_ = needle_nesting_depth_; + if (constraint_buffer_.size() >= 13 && + constraint_buffer_.compare(constraint_buffer_.size() - 13, 13, "\"arguments\":{") == 0) { + in_arguments_ = true; + arguments_depth_ = nesting_depth_; return; } - if (needle_in_arguments_ && - needle_nesting_depth_ == needle_arguments_depth_ && + if (in_arguments_ && + nesting_depth_ == arguments_depth_ && needle_at_arg_key_start()) { - needle_json_state_ = NeedleJsonState::IN_ARG_KEY; - needle_constrained_buf_.clear(); + region_ = Region::IN_ARG_KEY; + constrained_buf_.clear(); return; } if (ch == '"' && needle_is_value_string_start()) { - needle_in_string_value_ = true; + in_string_value_ = true; } } @@ -207,61 +307,105 @@ void ToolCallConstrainer::feed_needle_text(const std::string& text) { for (char ch : text) feed_needle_char(ch); } -void ToolCallConstrainer::init_needle_constraints() { - reset_needle_constraints(); - - needle_name_trie_ = std::make_unique(); - needle_param_tries_.clear(); - for (const auto& tool : tool_specs_) { - if (!tool.name.empty()) needle_insert_word(needle_name_trie_.get(), tool.name); - - auto param_root = std::make_unique(); - for (const auto& param_name : tool.parameter_names) { - if (!param_name.empty()) needle_insert_word(param_root.get(), param_name); +void ToolCallConstrainer::feed_gemma_char(char ch) { + if (state_ != State::GEMMA_NAME && state_ != State::GEMMA_ARGS) return; + + constraint_buffer_.push_back(ch); + + if (region_ == Region::IN_NAME) { + if (ch == '{') { + current_function_ = constrained_buf_; + constrained_buf_.clear(); + region_ = Region::FREE; + state_ = State::GEMMA_ARGS; + in_arguments_ = true; + nesting_depth_ = 1; + arguments_depth_ = nesting_depth_; + at_key_start_ = true; + rebuild_remaining_keys(); + } else { + constrained_buf_.push_back(ch); } - needle_param_tries_[tool.name] = std::move(param_root); + return; } - const uint32_t vocab_size = tokenizer_ ? tokenizer_->get_vocab_size() : 0; - if (needle_token_strings_.size() != vocab_size) { - needle_token_strings_.assign(vocab_size, ""); - needle_token_index_.clear(); - - const uint32_t eos = tokenizer_->get_eos_token(); - const uint32_t bos = tokenizer_->get_bos_token(); - const uint32_t unk = tokenizer_->get_unk_token(); - constexpr uint32_t pad = 0; - - for (uint32_t token_id = 0; token_id < vocab_size; ++token_id) { - if (token_id == pad || token_id == eos || token_id == bos || token_id == unk) continue; - std::string decoded = tokenizer_->decode({token_id}); - if (decoded.size() == 1) { - unsigned char b = static_cast(decoded[0]); - if (b >= 0x80) { - decoded.assign({static_cast(0xC0 | (b >> 6)), - static_cast(0x80 | (b & 0x3F))}); + if (region_ == Region::IN_ARG_KEY) { + if (ch == ':') { + emitted_keys_.insert(constrained_buf_); + active_enum_trie_ = nullptr; + if (escape_tag_valid_) { + auto fit = enum_tries_.find(current_function_); + if (fit != enum_tries_.end()) { + auto kit = fit->second.find(constrained_buf_); + if (kit != fit->second.end()) active_enum_trie_ = kit->second.get(); } } - needle_token_strings_[token_id] = decoded; - if (!decoded.empty()) needle_token_index_[decoded.front()].push_back(token_id); + await_enum_open_ = active_enum_trie_ != nullptr; + rebuild_remaining_keys(); + constrained_buf_.clear(); + region_ = Region::FREE; + } else { + constrained_buf_.push_back(ch); + } + return; + } + + if (in_string_value_) { + if (ends_with_escape_tag(constraint_buffer_)) { + in_string_value_ = false; + active_enum_trie_ = nullptr; + constrained_buf_.clear(); + } else if (active_enum_trie_) { + constrained_buf_.push_back(ch); } + return; + } + if (ends_with_escape_tag(constraint_buffer_)) { + in_string_value_ = true; + at_key_start_ = false; + await_enum_open_ = false; + constrained_buf_.clear(); + return; + } + if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r') return; + + if (at_key_start_ && ch != '}' && ch != ',') { + region_ = Region::IN_ARG_KEY; + constrained_buf_.clear(); + constrained_buf_.push_back(ch); + at_key_start_ = false; + return; + } + + if (ch == '{' || ch == '[') { + nesting_depth_++; + } else if (ch == '}' || ch == ']') { + nesting_depth_--; + if (ch == '}' && nesting_depth_ < arguments_depth_) { + in_arguments_ = false; + state_ = State::GEMMA_EXPECT_END; + forced_tag_progress_ = 0; + } + } else if (ch == ',' && nesting_depth_ == arguments_depth_) { + at_key_start_ = true; } } +void ToolCallConstrainer::feed_gemma_text(const std::string& text) { + for (char ch : text) feed_gemma_char(ch); +} + void ToolCallConstrainer::tokenize_grammar_elements() { if (!tokenizer_) return; open_brace_tokens_.clear(); close_brace_tokens_.clear(); - colon_tokens_.clear(); - comma_tokens_.clear(); init_common_tokens(); gemma_call_start_tokens_.clear(); gemma_call_end_tokens_.clear(); gemma_call_prefix_tokens_.clear(); - escape_tokens_.clear(); gemma_response_start_tokens_.clear(); call_start_sequence_ = tokenizer_->encode(call_start_tag_); @@ -274,10 +418,10 @@ void ToolCallConstrainer::tokenize_grammar_elements() { add_tokens_for_string("{", open_brace_tokens_); add_tokens_for_string("}", close_brace_tokens_); - add_tokens_for_string(":", colon_tokens_); - add_tokens_for_string(",", comma_tokens_); - tokenize_function_names(false); + std::vector escape_sequence = tokenizer_->encode(kEscapeTag); + escape_tag_valid_ = escape_sequence.size() == 1; + escape_tag_token_ = escape_tag_valid_ ? escape_sequence[0] : 0; } void ToolCallConstrainer::init(Config::ModelType model_type, @@ -285,21 +429,14 @@ void ToolCallConstrainer::init(Config::ModelType model_type, Tokenizer* tokenizer) { model_type_ = model_type; tool_specs_ = tools; - function_names_.clear(); - function_names_.reserve(tool_specs_.size()); - for (const auto& tool : tool_specs_) { - function_names_.push_back(tool.name); - } tokenizer_ = tokenizer; generated_text_.clear(); current_bias_.clear(); - brace_depth_ = 0; - in_argument_string_ = false; forced_tag_progress_ = 0; const bool model_supported = is_needle() || Config::is_gemma_family(model_type_); - active_ = model_supported && !function_names_.empty() && tokenizer != nullptr; + active_ = model_supported && !tool_specs_.empty() && tokenizer != nullptr; - reset_needle_constraints(); + reset_constraint_state(); state_ = is_needle() ? State::NEEDLE_START : State::GEMMA_START; if (Config::is_gemma3_family(model_type_)) { call_start_tag_ = ""; @@ -316,9 +453,7 @@ void ToolCallConstrainer::init(Config::ModelType model_type, } tokenize_grammar_elements(); - if (is_needle()) { - init_needle_constraints(); - } + build_constraint_tables(); compute_bias(); } @@ -354,49 +489,18 @@ void ToolCallConstrainer::update(uint32_t token_id, const std::string& decoded_t case State::GEMMA_EXPECT_CALL: if (generated_text_.find("call:") != std::string::npos) { - state_ = State::GEMMA_IN_FUNC_NAME; - generated_text_.clear(); - } - break; - - case State::GEMMA_IN_FUNC_NAME: - for (const auto& name : function_names_) { - if (generated_text_.find(name) != std::string::npos) { - state_ = State::GEMMA_EXPECT_BRACE; - generated_text_.clear(); - break; - } - } - break; - - case State::GEMMA_EXPECT_BRACE: - if (generated_text_.find("{") != std::string::npos) { - state_ = State::GEMMA_IN_ARGUMENTS; - brace_depth_ = 1; - in_argument_string_ = false; + state_ = State::GEMMA_NAME; generated_text_.clear(); + region_ = Region::IN_NAME; + constrained_buf_.clear(); + constraint_buffer_.clear(); } break; - case State::GEMMA_IN_ARGUMENTS: + case State::GEMMA_NAME: + case State::GEMMA_ARGS: + feed_gemma_text(decoded_text); generated_text_.clear(); - if (contains_odd_quote_tags(decoded_text)) { - in_argument_string_ = !in_argument_string_; - } - if (!in_argument_string_) { - for (char c : decoded_text) { - if (c == '{') brace_depth_++; - else if (c == '}') { - brace_depth_--; - if (brace_depth_ == 0) { - state_ = State::GEMMA_EXPECT_END; - in_argument_string_ = false; - forced_tag_progress_ = 0; - break; - } - } - } - } break; case State::GEMMA_EXPECT_END: @@ -423,56 +527,28 @@ void ToolCallConstrainer::compute_bias() { for (uint32_t t : backtick_tokens_) { current_bias_[t] = BLOCK_BIAS; } - for (uint32_t t : gemma_response_start_tokens_) { current_bias_[t] = BLOCK_BIAS; } } - if (is_needle()) { - if (needle_json_state_ == NeedleJsonState::FREE) return; - - const NeedleTrieNode* trie_node = nullptr; - if (needle_json_state_ == NeedleJsonState::IN_NAME) { - trie_node = needle_get_trie_node(needle_name_trie_.get(), needle_constrained_buf_); - } else if (needle_json_state_ == NeedleJsonState::IN_ARG_KEY) { - auto it = needle_param_tries_.find(needle_current_function_); - if (it != needle_param_tries_.end()) { - trie_node = needle_get_trie_node(it->second.get(), needle_constrained_buf_); - } - } - if (!trie_node) return; - - std::vector valid_tokens(needle_token_strings_.size(), false); - bool has_valid = false; - auto mark_valid_tokens = [&](char first_char) { - auto idx_it = needle_token_index_.find(first_char); - if (idx_it == needle_token_index_.end()) return; - for (uint32_t token_id : idx_it->second) { - if (!valid_tokens[token_id] && - needle_check_token_valid(needle_token_strings_[token_id], trie_node)) { - valid_tokens[token_id] = true; - has_valid = true; - } - } - }; - - for (const auto& [first_char, _] : trie_node->children) { - mark_valid_tokens(first_char); - } - if (trie_node->is_terminal) { - mark_valid_tokens('"'); - } - if (!has_valid) return; - - for (size_t token_id = 0; token_id < valid_tokens.size(); ++token_id) { - if (!valid_tokens[token_id]) { - current_bias_[static_cast(token_id)] = NEEDLE_BLOCK_BIAS; - } + if (region_ != Region::FREE) { + const TrieNode* node = nullptr; + char terminator = '"'; + if (region_ == Region::IN_NAME) { + node = trie_seek(name_trie_.get(), constrained_buf_); + if (!is_needle()) terminator = '{'; + } else { + const TrieNode* root = is_needle() ? current_param_trie() : remaining_key_trie_.get(); + node = trie_seek(root, constrained_buf_); + if (!is_needle()) terminator = ':'; } + mark_trie_bias(node, terminator, {}, -1, !is_needle()); return; } + if (is_needle()) return; + switch (state_) { case State::GEMMA_START: if (forced_tag_progress_ < call_start_sequence_.size()) { @@ -498,50 +574,24 @@ void ToolCallConstrainer::compute_bias() { } break; - case State::GEMMA_IN_FUNC_NAME: - for (uint32_t t : all_func_name_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_EXPECT_BRACE: - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_IN_ARGUMENTS: - if (!in_argument_string_) { - for (uint32_t t : colon_tokens_) { - current_bias_[t] = 10.0f; - } - for (uint32_t t : comma_tokens_) { - current_bias_[t] = 8.0f; + case State::GEMMA_ARGS: + if (await_enum_open_) { + current_bias_[escape_tag_token_] = FORCE_BIAS; + for (uint32_t t : gemma_call_end_tokens_) current_bias_[t] = BLOCK_BIAS; + for (uint32_t t : gemma_call_start_tokens_) current_bias_[t] = BLOCK_BIAS; + } else if (active_enum_trie_ && in_string_value_) { + mark_trie_bias(trie_seek(active_enum_trie_, constrained_buf_), '\0', {}, + static_cast(escape_tag_token_)); + } else if (at_key_start_) { + std::vector extra; + if (required_satisfied()) extra.push_back('}'); + mark_trie_bias(remaining_key_trie_.get(), ':', extra); + } else { + for (uint32_t t : gemma_call_end_tokens_) current_bias_[t] = BLOCK_BIAS; + for (uint32_t t : gemma_call_start_tokens_) current_bias_[t] = BLOCK_BIAS; + if (nesting_depth_ == arguments_depth_ && !in_string_value_ && !required_satisfied()) { + for (uint32_t t : close_brace_tokens_) current_bias_[t] = BLOCK_BIAS; } - for (uint32_t t : escape_tokens_) { - current_bias_[t] = 5.0f; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = 3.0f; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = 3.0f; - } - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : gemma_call_start_tokens_) { - current_bias_[t] = BLOCK_BIAS; } break; @@ -565,10 +615,8 @@ void ToolCallConstrainer::compute_bias() { void ToolCallConstrainer::reset() { generated_text_.clear(); current_bias_.clear(); - brace_depth_ = 0; - in_argument_string_ = false; forced_tag_progress_ = 0; - reset_needle_constraints(); + reset_constraint_state(); state_ = is_needle() ? State::NEEDLE_START : State::GEMMA_START; diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 34678a10f..46bcdf415 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -281,6 +281,7 @@ struct ChatMessage { struct ToolConstraintSpec { std::string name; std::vector parameter_names; + std::vector> parameter_enums; std::vector required_parameter_names; }; @@ -510,9 +511,8 @@ class ToolCallConstrainer { DONE, GEMMA_START, GEMMA_EXPECT_CALL, - GEMMA_IN_FUNC_NAME, - GEMMA_EXPECT_BRACE, - GEMMA_IN_ARGUMENTS, + GEMMA_NAME, + GEMMA_ARGS, GEMMA_EXPECT_END, NEEDLE_START }; @@ -530,16 +530,19 @@ class ToolCallConstrainer { bool is_active() const { return active_; } private: + enum class Region { FREE, IN_NAME, IN_ARG_KEY }; + struct TrieNode { + std::unordered_map> children; + bool is_terminal = false; + }; + bool active_ = false; State state_ = State::GEMMA_START; Config::ModelType model_type_ = Config::ModelType::GEMMA4; Tokenizer* tokenizer_ = nullptr; std::vector tool_specs_; - std::vector function_names_; std::string generated_text_; - int brace_depth_ = 0; - bool in_argument_string_ = false; std::string call_start_tag_; std::string call_end_tag_; @@ -548,65 +551,65 @@ class ToolCallConstrainer { std::vector call_end_sequence_; size_t forced_tag_progress_ = 0; - std::unordered_set all_func_name_tokens_; - std::unordered_map> func_name_sequences_; - std::unordered_set gemma_call_start_tokens_; std::unordered_set gemma_call_end_tokens_; std::unordered_set gemma_response_start_tokens_; std::unordered_set gemma_call_prefix_tokens_; - std::unordered_set escape_tokens_; std::unordered_set backtick_tokens_; std::unordered_set open_brace_tokens_; std::unordered_set close_brace_tokens_; - std::unordered_set colon_tokens_; - std::unordered_set comma_tokens_; std::unordered_map current_bias_; void compute_bias(); void tokenize_grammar_elements(); void add_tokens_for_string(const std::string& str, std::unordered_set& token_set); - void add_tokens_for_prefix_string(const std::string& prefix, std::unordered_set& token_set); void advance_forced_tag(const std::vector& sequence, uint32_t token_id); - void tokenize_function_names(bool quote_names); void init_common_tokens(); bool is_needle() const { return model_type_ == Config::ModelType::NEEDLE; } - enum class NeedleJsonState { - FREE, - IN_NAME, - IN_ARG_KEY - }; - struct NeedleTrieNode { - std::unordered_map> children; - bool is_terminal = false; - }; - NeedleJsonState needle_json_state_ = NeedleJsonState::FREE; - std::string needle_buffer_; - std::string needle_constrained_buf_; - std::string needle_current_function_; - bool needle_in_arguments_ = false; - int needle_arguments_depth_ = 0; - int needle_nesting_depth_ = 0; - bool needle_in_string_value_ = false; - bool needle_prev_char_escape_ = false; - std::unique_ptr needle_name_trie_; - std::unordered_map> needle_param_tries_; - std::vector needle_token_strings_; - std::unordered_map> needle_token_index_; - - void init_needle_constraints(); - void reset_needle_constraints(); + Region region_ = Region::FREE; + std::string constraint_buffer_; + std::string constrained_buf_; + std::string current_function_; + bool in_arguments_ = false; + int arguments_depth_ = 0; + int nesting_depth_ = 0; + bool in_string_value_ = false; + bool prev_char_escape_ = false; + bool at_key_start_ = false; + bool await_enum_open_ = false; + const TrieNode* active_enum_trie_ = nullptr; + std::unordered_set emitted_keys_; + std::unique_ptr name_trie_; + std::unordered_map> param_tries_; + std::unordered_map>> enum_tries_; + std::unique_ptr remaining_key_trie_; + std::vector token_strings_; + std::unordered_map> token_index_; + uint32_t escape_tag_token_ = 0; + bool escape_tag_valid_ = false; + + void build_constraint_tables(); + void reset_constraint_state(); + void trie_insert(TrieNode* root, const std::string& word); + const TrieNode* trie_seek(const TrieNode* root, const std::string& prefix) const; + bool trie_token_valid(const std::string& token_text, const TrieNode* node, char terminator) const; + void mark_trie_bias(const TrieNode* node, char terminator, const std::vector& extra_allowed, + int32_t terminal_token = -1, bool exact_terminator = false); + const TrieNode* current_param_trie() const; + void rebuild_remaining_keys(); + bool required_satisfied() const; + void feed_needle_text(const std::string& text); void feed_needle_char(char ch); bool needle_at_arg_key_start() const; bool needle_is_value_string_start() const; - void needle_insert_word(NeedleTrieNode* root, const std::string& word); - const NeedleTrieNode* needle_get_trie_node(const NeedleTrieNode* root, const std::string& prefix) const; - bool needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const; + + void feed_gemma_text(const std::string& text); + void feed_gemma_char(char ch); }; class Model { diff --git a/python/tests/test_tool_constraint_state.py b/python/tests/test_tool_constraint_state.py index 9fe53d807..7a1096855 100644 --- a/python/tests/test_tool_constraint_state.py +++ b/python/tests/test_tool_constraint_state.py @@ -13,20 +13,54 @@ WEIGHTS / "gemma-4-e2b-it", ) -GET_WEATHER = { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City name"}, - }, - "required": ["location"], +def _tool(name, description, properties, required): + return { + "type": "function", + "function": { + "name": name, + "description": description, + "parameters": {"type": "object", "properties": properties, "required": required}, }, - }, -} + } + + +GET_WEATHER = _tool("get_weather", "Get the current weather for a location", + {"location": {"type": "string", "description": "City name"}}, ["location"]) +SET_ALARM = _tool("set_alarm", "Set an alarm for a time", + {"time": {"type": "string", "description": "HH:MM"}}, ["time"]) +SEND = _tool("send", "Send a quick ping (no body) to a user id", + {"user": {"type": "string"}}, ["user"]) +SEND_MESSAGE = _tool("send_message", "Send a text message with a body to a contact", + {"recipient": {"type": "string"}, "body": {"type": "string"}}, ["recipient", "body"]) +SEARCH = _tool("search", "Search local notes", {"query": {"type": "string"}}, ["query"]) +SEARCH_WEB = _tool("search_web", "Search the public web", {"query": {"type": "string"}}, ["query"]) +GET = _tool("get", "Get a raw value by key", {"key": {"type": "string"}}, ["key"]) +GET_USER = _tool("get_user", "Fetch a user profile by id", {"id": {"type": "string"}}, ["id"]) +PING_SERVER = _tool("ping_server", "Check server health, takes no arguments", {}, []) +WEATHER_UNIT = _tool("get_weather", "Get weather for a city in a unit", + {"location": {"type": "string", "description": "city"}, + "unit": {"type": "string", "description": "temperature unit", + "enum": ["celsius", "fahrenheit"]}}, + ["location", "unit"]) +CREATE_EVENT = _tool("create_event", "Create a calendar event", + {"title": {"type": "string"}, "date": {"type": "string", "description": "YYYY-MM-DD"}, + "location": {"type": "string"}}, + ["title", "date"]) + + +def _forced_calls(model, tools, user): + result = cactus_complete( + model, + [ + {"role": "system", "content": "You are a helpful assistant that can use tools."}, + {"role": "user", "content": user}, + ], + {"force_tools": True, "max_tokens": 128, "temperature": 0}, + tools, + ) + assert result.get("success"), result + calls = result.get("function_calls") or [] + return [c if isinstance(c, dict) else json.loads(c) for c in calls] @pytest.fixture(scope="module") @@ -67,3 +101,124 @@ def test_turn_after_force_tools_is_not_degenerate(model): response = followup.get("response", "") assert response.strip() assert "<|tool_call>" not in response + + +def test_prefix_name_is_not_shadowed(model): + calls = _forced_calls(model, [SEND, SEND_MESSAGE], "Message my contact Alice: running late, start without me.") + assert calls + assert calls[0]["name"] == "send_message" + + +@pytest.mark.parametrize("tools, user, want", [ + ([SEARCH, SEARCH_WEB], "Search the public web for the best ramen in Osaka.", "search_web"), + ([GET, GET_USER], "Fetch the user profile for account 42.", "get_user"), +]) +def test_prefix_pairs_reach_longer_name(model, tools, user, want): + calls = _forced_calls(model, tools, user) + assert calls + assert calls[0]["name"] == want + + +def test_shorter_prefix_name_still_reachable(model): + calls = _forced_calls(model, [SEND, SEND_MESSAGE], "Ping user u_42.") + assert calls + assert calls[0]["name"] == "send" + + +def test_forced_name_is_always_a_declared_tool(model): + tools = [GET_WEATHER, SET_ALARM] + declared = {t["function"]["name"] for t in tools} + calls = _forced_calls(model, tools, "What's the weather in Tokyo right now?") + assert calls + assert calls[0]["name"] in declared + + +def test_no_argument_function_emits_empty_object(model): + calls = _forced_calls(model, [PING_SERVER], "Check whether the server is healthy.") + assert calls + assert calls[0]["name"] == "ping_server" + assert calls[0]["arguments"] == {} + + +def test_argument_keys_are_declared_parameters(model): + calls = _forced_calls(model, [GET_WEATHER], "What's the weather in Berlin?") + assert calls + assert calls[0]["name"] == "get_weather" + assert set(calls[0]["arguments"]).issubset({"location"}) + + +@pytest.mark.parametrize("user", [ + "What's the weather in Tokyo in kelvin?", + "Weather in Paris, in Kelvin please.", + "Weather in Berlin.", +]) +def test_enum_argument_restricted_to_allowed_values(model, user): + calls = _forced_calls(model, [WEATHER_UNIT], user) + assert calls + assert calls[0]["arguments"].get("unit") in ("celsius", "fahrenheit") + + +@pytest.mark.parametrize("user", [ + "Make a calendar event.", + "Add an event called Standup.", + "Schedule a dentist appointment.", +]) +def test_required_arguments_are_always_present(model, user): + calls = _forced_calls(model, [CREATE_EVENT], user) + assert calls + args = calls[0]["arguments"] + assert "title" in args and "date" in args + + +def test_enum_with_shared_prefixes_can_reach_longest_value(model): + tool = _tool("pick_plan", "pick a plan", + {"plan": {"type": "string", "enum": ["pro", "pro_plus", "pro_plus_max"]}}, ["plan"]) + calls = _forced_calls(model, [tool], "Pick the pro plus max plan.") + assert calls + assert calls[0]["arguments"].get("plan") in ("pro", "pro_plus", "pro_plus_max") + + +def test_enum_value_containing_spaces(model): + tool = _tool("move_card", "move a kanban card", + {"card": {"type": "string"}, + "column": {"type": "string", "enum": ["to do", "in progress", "done"]}}, + ["card", "column"]) + calls = _forced_calls(model, [tool], "Move the login card to in progress.") + assert calls + assert calls[0]["arguments"].get("column") in ("to do", "in progress", "done") + + +def test_single_value_enum_is_forced(model): + tool = _tool("finalize", "finalize", {"status": {"type": "string", "enum": ["confirmed"]}}, ["status"]) + calls = _forced_calls(model, [tool], "Finalize it however you want.") + assert calls + assert calls[0]["arguments"].get("status") == "confirmed" + + +def test_prefix_parameter_names_both_required_present(model): + tool = _tool("make_range", "range", {"to": {"type": "string"}, "total": {"type": "string"}}, ["to", "total"]) + calls = _forced_calls(model, [tool], "Make a range up to 10 with a total of 100.") + assert calls + assert {"to", "total"} <= set(calls[0]["arguments"]) + + +def test_many_required_arguments_all_present(model): + tool = _tool("register", "register a device", + {k: {"type": "string"} for k in ("name", "os", "owner", "ip", "room")}, + ["name", "os", "owner", "ip", "room"]) + calls = _forced_calls(model, [tool], "Register a device.") + assert calls + assert {"name", "os", "owner", "ip", "room"} <= set(calls[0]["arguments"]) + + +def test_prefix_tool_with_required_enum_on_longer_name(model): + send = _tool("send", "ping a user", {"user": {"type": "string"}}, ["user"]) + send_message = _tool("send_message", "send a message", + {"to": {"type": "string"}, "body": {"type": "string"}, + "urgency": {"type": "string", "enum": ["normal", "urgent"]}}, + ["to", "body", "urgency"]) + calls = _forced_calls(model, [send, send_message], "Send an urgent message to Alice saying the server is down.") + assert calls + assert calls[0]["name"] == "send_message" + assert calls[0]["arguments"].get("urgency") in ("normal", "urgent") + assert {"to", "body", "urgency"} <= set(calls[0]["arguments"]) From 2a48b6c1d9330d7578307cd99f5182437a9e8447 Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 03:52:49 +0200 Subject: [PATCH 09/36] fixes + speedups Signed-off-by: jakmro --- cactus-engine/cactus_engine.h | 1 - cactus-engine/src/engine.h | 12 + cactus-engine/src/model.cpp | 180 ++++++- cactus-engine/tests/test_llm.cpp | 2 +- cactus-engine/tests/test_utils.h | 1 - cactus-graph/cactus_graph.h | 9 +- cactus-graph/src/execute.cpp | 1 + cactus-graph/src/gpu_fused.cpp | 510 ++++++++++++------ cactus-kernels/metal_backend.h | 34 +- cactus-kernels/src/cactus_kernels.metal | 620 ++++++++++++++++++++-- cactus-kernels/src/metal_backend.mm | 410 ++++++++++++-- cactus-kernels/src/metal_backend_stub.cpp | 25 +- 12 files changed, 1527 insertions(+), 278 deletions(-) diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index 18ee07012..5dacb0bf6 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -34,7 +34,6 @@ CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); -// backend: "auto", "cpu", or "metal". Returns 0 on success, -1 on failure CACTUS_FFI_EXPORT int cactus_set_backend(const char* backend); CACTUS_FFI_EXPORT int cactus_complete( diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 46bcdf415..515f4e37a 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -845,6 +846,9 @@ class Model { size_t chunk_tokens, const std::vector& tokens, size_t processed, size_t start_position); void run_full_context_text(); + void prepare_sampling_context(float repetition_penalty); + uint32_t sample_component_logits(Component& comp, float temperature, float top_p, size_t top_k, + float min_p, bool greedy, float* out_uncertainty); uint32_t argmax_component_logits(Component& comp, size_t logit_row = std::numeric_limits::max(), float* out_uncertainty = nullptr); uint32_t argmax_logits_at(const BufferDesc& desc, void* ptr, size_t row_off, float* out_uncertainty); @@ -942,6 +946,14 @@ class Model { static constexpr size_t MAX_TOKEN_HISTORY = 128; std::vector token_history_; + std::vector samp_recent_; + std::vector samp_bias_ids_; + std::vector samp_bias_vals_; + float samp_penalty_ = 1.0f; + bool samp_ctx_active_ = false; + std::mt19937 sample_rng_{std::random_device{}()}; + FusedEmbedCtx fused_embed_ctx_; + int ple_probe_state_ = 0; ToolCallConstrainer tool_constrainer_; std::unordered_map vocab_bias_; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 6f2352b1f..34e49cc4d 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -602,11 +602,6 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, bound.insert(comp); } - // Prewarm the prefill executor. The generic graph executor allocates every intermediate GPU - // buffer (gpu_persistent_acts_) lazily on the FIRST execute() — a ~190ms one-time cost that - // otherwise lands in the first real prefill chunk, inflating TTFT (dominant at short prompts and - // the reason pp512 measured slower than pp2048). Run each prefill graph once on zeroed inputs - // here, then reset the KV cache states the warmup dirtied so real inference starts clean. if (decoder_prefill_ && decoder_prefill_->graph && std::getenv("CACTUS_NO_PREFILL_WARM") == nullptr) { try { @@ -911,15 +906,12 @@ void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { } const bool fused = cactus_backend_fused(); if (fused) { - enum class PleProbe { Untried, Available, Unavailable }; - static PleProbe probe = PleProbe::Untried; - static FusedEmbedCtx ctx; - if (probe == PleProbe::Untried) - probe = encoder_->graph->extract_ple_pathway(ctx) ? PleProbe::Available : PleProbe::Unavailable; - if (probe == PleProbe::Available) { - ctx.token_id = static_cast(token_id); - ctx.position = static_cast(position); - cactus_graph_set_fused_embed(&ctx); + if (ple_probe_state_ == 0) + ple_probe_state_ = encoder_->graph->extract_ple_pathway(fused_embed_ctx_) ? 1 : 2; + if (ple_probe_state_ == 1) { + fused_embed_ctx_.token_id = static_cast(token_id); + fused_embed_ctx_.position = static_cast(position); + cactus_graph_set_fused_embed(&fused_embed_ctx_); bool ok = decoder_->graph->execute_gpu_fused(); cactus_graph_set_fused_embed(nullptr); if (ok) { maybe_capture_handoff_probe_hidden(*decoder_); return; } @@ -2032,6 +2024,143 @@ uint32_t Model::argmax_last_logits(float* out_uncertainty) { return argmax_component_logits(*decoder_, std::numeric_limits::max(), out_uncertainty); } +void Model::prepare_sampling_context(float repetition_penalty) { + samp_recent_.clear(); + samp_bias_ids_.clear(); + samp_bias_vals_.clear(); + samp_penalty_ = repetition_penalty; + constexpr size_t kPenaltyWindow = 64; + size_t start = token_history_.size() > kPenaltyWindow ? token_history_.size() - kPenaltyWindow : 0; + for (size_t i = start; i < token_history_.size(); ++i) { + uint32_t id = token_history_[i]; + bool dup = false; + for (uint32_t r : samp_recent_) if (r == id) { dup = true; break; } + if (!dup) samp_recent_.push_back(id); + } + for (const auto& kv : tool_constrainer_.get_bias()) { samp_bias_ids_.push_back(kv.first); samp_bias_vals_.push_back(kv.second); } + for (const auto& kv : vocab_bias_) { samp_bias_ids_.push_back(kv.first); samp_bias_vals_.push_back(kv.second); } + samp_ctx_active_ = (repetition_penalty != 1.0f && !samp_recent_.empty()) || + !samp_bias_ids_.empty() || suppressed_token_id_ >= 0; + if (samp_ctx_active_) { + cactus_graph_set_sampling(samp_recent_.data(), (int)samp_recent_.size(), repetition_penalty, + samp_bias_ids_.data(), samp_bias_vals_.data(), (int)samp_bias_ids_.size(), + suppressed_token_id_); + } else { + cactus_graph_clear_sampling(); + } +} + +uint32_t Model::sample_component_logits(Component& comp, float temperature, float top_p, size_t top_k, + float min_p, bool greedy, float* out_uncertainty) { + size_t out_node = static_cast(comp.output_node_ids.empty() ? 0 : comp.output_node_ids[0]); + const auto& desc = comp.graph->get_output_buffer(out_node); + void* ptr = comp.graph->get_output(out_node); + size_t vocab = desc.shape.empty() ? 0 : desc.shape.back(); + size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; + size_t row = seq > 0 ? seq - 1 : 0; + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + row = std::min(last_logit_position_, seq > 0 ? seq - 1 : 0); + } + size_t off = row * vocab; + if (vocab == 0 || (desc.precision != Precision::FP16 && desc.precision != Precision::FP32)) { + return argmax_component_logits(comp, std::numeric_limits::max(), out_uncertainty); + } + const bool fp16 = desc.precision == Precision::FP16; + __fp16* h = fp16 ? static_cast<__fp16*>(ptr) + off : nullptr; + float* f = fp16 ? nullptr : static_cast(ptr) + off; + auto get = [&](size_t i) -> float { return fp16 ? (float)h[i] : f[i]; }; + auto put = [&](size_t i, float v) { if (fp16) h[i] = (__fp16)v; else f[i] = v; }; + + if (samp_ctx_active_ && !cactus_graph_gpu_adjusted()) { + if (samp_penalty_ != 1.0f) { + for (uint32_t id : samp_recent_) { + if (id >= vocab) continue; + float v = get(id); + put(id, v > 0.0f ? v / samp_penalty_ : v * samp_penalty_); + } + } + for (size_t i = 0; i < samp_bias_ids_.size(); ++i) { + uint32_t id = samp_bias_ids_[i]; + if (id < vocab) put(id, get(id) + samp_bias_vals_[i]); + } + if (suppressed_token_id_ >= 0 && (size_t)suppressed_token_id_ < vocab) + put((size_t)suppressed_token_id_, -65504.0f); + } + + if (greedy) { + uint32_t gidx; float gbest, gsecond; + if ((!samp_ctx_active_ || cactus_graph_gpu_adjusted()) && + cactus_graph_gpu_argmax(&gidx, &gbest, &gsecond)) { + if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(gbest, gsecond); + return gidx; + } + uint32_t best = 0; + float bv = -std::numeric_limits::infinity(), sv = bv; + for (size_t i = 0; i < vocab; ++i) { + float v = get(i); + if (v > bv) { sv = bv; bv = v; best = (uint32_t)i; } + else if (v > sv) sv = v; + } + if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(bv, sv); + return best; + } + + size_t K = std::min(std::max(top_k, 1), 512); + std::vector> cand; + cand.reserve(2 * K + 16); + float kmin = -std::numeric_limits::infinity(); + for (size_t i = 0; i < vocab; ++i) { + float v = get(i); + if (v <= kmin) continue; + cand.emplace_back(v, (uint32_t)i); + if (cand.size() >= 2 * K) { + std::nth_element(cand.begin(), cand.begin() + (K - 1), cand.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + cand.resize(K); + kmin = cand.back().first; + for (const auto& c : cand) if (c.first < kmin) kmin = c.first; + } + } + std::sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); + if (cand.size() > K) cand.resize(K); + if (cand.empty()) return 0; + if (out_uncertainty) { + float sv = cand.size() > 1 ? cand[1].first : -std::numeric_limits::infinity(); + *out_uncertainty = uncertainty_from_margin(cand[0].first, sv); + } + if (cand.size() == 1) return cand[0].second; + + float t = std::max(temperature, 1e-3f); + float maxl = cand[0].first; + std::vector probs(cand.size()); + double denom = 0.0; + for (size_t i = 0; i < cand.size(); ++i) { + probs[i] = std::exp((cand[i].first - maxl) / t); + denom += probs[i]; + } + size_t keep = cand.size(); + if (top_p > 0.0f && top_p < 1.0f) { + double cum = 0.0; + for (size_t i = 0; i < cand.size(); ++i) { + cum += probs[i] / denom; + if (cum >= top_p) { keep = i + 1; break; } + } + } + if (min_p > 0.0f) { + float cut = min_p * probs[0]; + while (keep > 1 && probs[keep - 1] < cut) --keep; + } + double mass = 0.0; + for (size_t i = 0; i < keep; ++i) mass += probs[i]; + double r = std::uniform_real_distribution(0.0, mass)(sample_rng_); + double cum = 0.0; + for (size_t i = 0; i < keep; ++i) { + cum += probs[i]; + if (r <= cum) return cand[i].second; + } + return cand[keep - 1].second; +} + bool Model::prefill_and_sample_first_token(const std::vector& tokens, uint32_t& out_token, float* out_uncertainty) { reset_prefill_stats(); @@ -3039,10 +3168,15 @@ void Model::prefill_with_media(const std::vector& tokens, (void)profile_file; } -uint32_t Model::decode(const std::vector& tokens, float /*temperature*/, float /*top_p*/, - size_t /*top_k*/, const std::string& /*profile_file*/, float* out_entropy, - float /*min_p*/, float /*repetition_penalty*/) { +uint32_t Model::decode(const std::vector& tokens, float temperature, float top_p, + size_t top_k, const std::string& /*profile_file*/, float* out_entropy, + float min_p, float repetition_penalty) { if (tokens.empty()) return 0; + float temp = temperature < 0.0f ? config_.default_temperature : temperature; + float tp = (top_p <= 0.0f || top_p > 1.0f) ? config_.default_top_p : top_p; + size_t tk = top_k == 0 ? config_.default_top_k : top_k; + const bool greedy = temp <= 0.011f; + prepare_sampling_context(repetition_penalty); if (decode_route_ == DecodeRoute::ENCODER_CROSS_KV_STEP) { if (!encoder_cross_kv_ready_ && encoder_cross_kv_source_kind_ == "text_tokens") { std::vector source_tokens = tokens; @@ -3074,7 +3208,11 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* run_full_context_text(); cache_total_seq_len_ = context_tokens_.size(); cache_token_ids_ = context_tokens_; - uint32_t result = argmax_last_logits(out_entropy); + uint32_t result = (greedy && !samp_ctx_active_) + ? argmax_last_logits(out_entropy) + : sample_component_logits(*decoder_, temp, tp, tk, min_p, greedy, out_entropy); + cactus_graph_clear_sampling(); + samp_ctx_active_ = false; record_sampled_token(result); return result; } @@ -3085,7 +3223,11 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* cache_total_seq_len_ += tokens.size(); cache_token_ids_.insert(cache_token_ids_.end(), tokens.begin(), tokens.end()); maybe_roll_compact(); - uint32_t result = argmax_last_logits(out_entropy); + uint32_t result = (greedy && !samp_ctx_active_) + ? argmax_last_logits(out_entropy) + : sample_component_logits(*decoder_, temp, tp, tk, min_p, greedy, out_entropy); + cactus_graph_clear_sampling(); + samp_ctx_active_ = false; record_sampled_token(result); return result; } diff --git a/cactus-engine/tests/test_llm.cpp b/cactus-engine/tests/test_llm.cpp index 58cb1eaad..93ce6d9b2 100644 --- a/cactus-engine/tests/test_llm.cpp +++ b/cactus-engine/tests/test_llm.cpp @@ -437,7 +437,7 @@ struct LengthTokenizer : public Tokenizer { bool test_tool_constraint_clear_releases_bias() { LengthTokenizer tok; ToolCallConstrainer constrainer; - constrainer.init(Config::ModelType::GEMMA4, {{"get_weather", {"location"}, {"location"}}}, &tok); + constrainer.init(Config::ModelType::GEMMA4, {{"get_weather", {"location"}, {}, {"location"}}}, &tok); if (constrainer.get_bias().empty()) { std::cerr << " expected bias after activating init\n"; return false; diff --git a/cactus-engine/tests/test_utils.h b/cactus-engine/tests/test_utils.h index ab7c96c9f..04cb792fe 100644 --- a/cactus-engine/tests/test_utils.h +++ b/cactus-engine/tests/test_utils.h @@ -27,7 +27,6 @@ class TestRunner { int total_count_; }; -// Select the inference backend from CACTUS_TEST_BACKEND ("auto"/"cpu"/"metal"). Applied once. void apply_backend(); } diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 6846179d7..795b8678b 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -467,11 +467,18 @@ struct FusedEmbedCtx { void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx); bool cactus_graph_gpu_argmax(uint32_t* idx, float* best, float* second); +void cactus_graph_mark_unadjusted(); +void cactus_graph_on_destroy(const void* graph); +void cactus_graph_set_sampling(const uint32_t* recent, int n_recent, float rep_penalty, + const uint32_t* bias_ids, const float* bias_vals, int n_bias, + long long suppressed); +void cactus_graph_clear_sampling(); +bool cactus_graph_gpu_adjusted(); class CactusGraph { public: CactusGraph(); - ~CactusGraph() = default; + ~CactusGraph(); CactusGraph(const CactusGraph&) = delete; CactusGraph& operator=(const CactusGraph&) = delete; CactusGraph(CactusGraph&&) noexcept = default; diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 86244f7e6..eb5f56300 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -748,6 +748,7 @@ static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const nod } void CactusGraph::execute(const std::string& profile_file) { + cactus_graph_mark_unadjusted(); BufferPool& pool = buffer_pool_; const size_t n = nodes_.size(); infer_shapes(); diff --git a/cactus-graph/src/gpu_fused.cpp b/cactus-graph/src/gpu_fused.cpp index 6f527a0aa..a0d6470ad 100644 --- a/cactus-graph/src/gpu_fused.cpp +++ b/cactus-graph/src/gpu_fused.cpp @@ -15,9 +15,21 @@ namespace { constexpr int NL = 35, HID = 1536, NQH = 8, PLE_DIM = 256; constexpr float RMS_EPS = 1e-6f, GATE_SCALE = 0.015625f, SOFTCAP = 30.0f; -// Fuse per-head rmsnorm+rope for q/k and fold the per-layer residual scalar into the post-PLE norm+add. -// Cuts non-gemv decode GPU work ~3% (Amdahl-capped to ~1% of total since gemv dominates). Default on. static const bool g_fuse_nr = (std::getenv("CACTUS_NO_FUSE_NR") == nullptr); +static const unsigned g_mega_mask = [](){ + const char* e = std::getenv("CACTUS_MEGA"); + if (std::getenv("CACTUS_NO_MEGA")) return 0u; + return e ? (unsigned)std::strtoul(e, nullptr, 0) : 0x38u; +}(); +static const bool g_mega_qkv = (g_mega_mask & 1u), g_mega_o = (g_mega_mask & 2u), + g_mega_gu = (g_mega_mask & 4u), g_mega_ple = (g_mega_mask & 8u), + g_mega_pj = (g_mega_mask & 16u), g_mega_dswt = (g_mega_mask & 32u), + g_mega_down = (g_mega_mask & 64u); +static const bool g_fattn = (std::getenv("CACTUS_NO_FATTN") == nullptr); +static const int g_flush_layers = [](){ + const char* e = std::getenv("CACTUS_FLUSH_LAYERS"); + return e ? std::atoi(e) : 12; +}(); struct LayerW { CactusQuantMatrix q, o, ple_down, ple_up, gate, up, down; @@ -44,6 +56,7 @@ struct Arena { struct GpuModel { bool init = false; + const void* owner = nullptr; LayerW L[NL]; size_t kc_idx[15], vc_idx[15]; const void *vnorm, *vnorm_g; @@ -54,7 +67,7 @@ struct GpuModel { const void *final_norm; CactusQuantMatrix lm_head; void *cos_s, *sin_s, *cos_g, *sin_g; - void *argmax_buf = nullptr; + void *amax = nullptr; Arena arena; }; static GpuModel G; @@ -64,6 +77,42 @@ static const float* g_gpu_argmax_buf = nullptr; } +static bool g_last_step_adjusted = false; + +void cactus_graph_mark_unadjusted() { + g_last_step_adjusted = false; +} + +struct GSampling { + bool active = false; + float rep_penalty = 1.0f; + std::vector recent; + std::vector bias_ids; + std::vector bias_vals; + long long suppressed = -1; +}; +static GSampling g_samp; +static bool g_step_adjusted = false; + +void cactus_graph_set_sampling(const uint32_t* recent, int n_recent, float rep_penalty, + const uint32_t* bias_ids, const float* bias_vals, int n_bias, + long long suppressed) { + g_samp.active = true; + g_samp.rep_penalty = rep_penalty; + g_samp.recent.assign(recent, recent + (n_recent > 0 ? n_recent : 0)); + g_samp.bias_ids.assign(bias_ids, bias_ids + (n_bias > 0 ? n_bias : 0)); + g_samp.bias_vals.assign(bias_vals, bias_vals + (n_bias > 0 ? n_bias : 0)); + g_samp.suppressed = suppressed; +} +void cactus_graph_clear_sampling() { g_samp.active = false; } +bool cactus_graph_gpu_adjusted() { return g_last_step_adjusted; } + +static inline bool sampling_needs_adjust() { + return g_samp.active && + ((g_samp.rep_penalty != 1.0f && !g_samp.recent.empty()) || + !g_samp.bias_ids.empty() || g_samp.suppressed >= 0); +} + bool cactus_graph_gpu_argmax(uint32_t* idx, float* best, float* second) { if (!g_gpu_argmax_valid || !g_gpu_argmax_buf) return false; g_gpu_argmax_valid = false; @@ -147,9 +196,14 @@ bool CactusGraph::execute_gpu_fused() { auto sh = [&](size_t bytes){ void* p = cactus_metal_alloc_shared(bytes); if (p) std::memset(p,0,bytes); return p; }; G.cos_s = sh(256*2); G.sin_s = sh(256*2); G.cos_g = sh(512*2); G.sin_g = sh(512*2); - G.argmax_buf = sh(3*sizeof(float)); + G.amax = sh(3*sizeof(float)); + G.owner = this; G.init = true; } + if (G.owner != this) { + G.init = false; + return false; + } const bool fold = g_fe.ok; @@ -170,154 +224,304 @@ bool CactusGraph::execute_gpu_fused() { if (G.L[l].global && km[0] + 1 > km[1]) return false; } - G.arena.reset(); - auto fresh = [&](size_t elems){ return G.arena.fresh(elems * 2); }; cactus_metal_session_begin(); - cactus_metal_set_active(true); + cactus_metal_session_set_concurrent(true); + + auto encode_step = [&](int pos_, uint32_t tok, void** lg_out) -> int { + G.arena.reset(); + auto fresh = [&](size_t elems){ return G.arena.fresh(elems * 2); }; + + void* h = fresh(HID); + const void* pleBase; + if (fold) { + const int PK = (int)g_fe.proj.N; + void* pe = fresh(g_fe.ple.K); + cactus_metal_group_begin(); + bool e1 = cactus_metal_encode_embedding_ortho(h, tok, &G.lm_head, g_fe.emb_scale); + bool e2 = cactus_metal_encode_embedding_hadamard(pe, tok, &g_fe.ple); + cactus_metal_group_end(); + if (!e1 || !e2) return 0; + void* pa = fresh(PK); cactus_metal_encode_scalar(2, pa, pe, PK, g_fe.ple_scale); + void* pj = fresh(PK); + const CactusQuantMatrix* Wpj[1]={&g_fe.proj}; void* ypj[1]={pj}; + if (!(g_mega_pj && cactus_metal_encode_gemv_mega(ypj, h, nullptr, 0.f, Wpj, 1, nullptr)) + && !cactus_metal_encode_quant_matmul(pj, h, &g_fe.proj)) return 0; + void* pjs= fresh(PK); cactus_metal_encode_scalar(2, pjs, pj, PK, g_fe.proj_scale); + void* plt= fresh(PK); + if (!cactus_metal_encode_rms_norm_add_scale(plt, pjs, g_fe.rms_weight, pa, NL, PLE_DIM, g_fe.rms_eps, g_fe.final_scale)) { + void* psum=fresh(PK); cactus_metal_encode_rms_norm_add(psum, pjs, g_fe.rms_weight, pa, NL, PLE_DIM, g_fe.rms_eps); + cactus_metal_encode_scalar(2, plt, psum, PK, g_fe.final_scale); + } + pleBase = plt; + } else { + std::memcpy(h, G.hidden, HID*2); + pleBase = G.ple; + } - void* h = fresh(HID); - const void* pleBase; - if (fold) { - auto bail = [&](){ cactus_metal_set_active(false); cactus_metal_session_end(); return false; }; - const uint32_t tok = (uint32_t)g_fe.token_id; - const int PK = (int)g_fe.proj.N; - - void* hr = fresh(HID); - if (!cactus_metal_encode_embedding_ortho(hr, tok, &G.lm_head)) return bail(); - cactus_metal_encode_scalar(2, h, hr, HID, g_fe.emb_scale); - - void* pe = fresh(g_fe.ple.K); - if (!cactus_metal_encode_embedding_hadamard(pe, tok, &g_fe.ple)) return bail(); - void* pa = fresh(PK); cactus_metal_encode_scalar(2, pa, pe, PK, g_fe.ple_scale); - void* pj = fresh(PK); - if (!cactus_metal_encode_quant_matmul(pj, h, &g_fe.proj)) return bail(); - void* pjs= fresh(PK); cactus_metal_encode_scalar(2, pjs, pj, PK, g_fe.proj_scale); - void* psum=fresh(PK); cactus_metal_encode_rms_norm_add(psum, pjs, g_fe.rms_weight, pa, NL, PLE_DIM, g_fe.rms_eps); - void* plt= fresh(PK); cactus_metal_encode_scalar(2, plt, psum, PK, g_fe.final_scale); - pleBase = plt; - } else { - std::memcpy(h, G.hidden, HID*2); - pleBase = G.ple; - } + auto gather = [&](const void* tbl, void* dst, int hd){ std::memcpy(dst, (const char*)tbl + (size_t)pos_*hd*2, hd*2); }; + gather(G.cosL, G.cos_s, 256); gather(G.sinL, G.sin_s, 256); + gather(G.cosG, G.cos_g, 512); gather(G.sinG, G.sin_g, 512); - auto gather = [&](const void* tbl, void* dst, int hd){ std::memcpy(dst, (const char*)tbl + (size_t)pos*hd*2, hd*2); }; - gather(G.cosL, G.cos_s, 256); gather(G.sinL, G.sin_s, 256); - gather(G.cosG, G.cos_g, 512); gather(G.sinG, G.sin_g, 512); + void* xn = fresh(HID); + cactus_metal_encode_rms_norm(xn, h, G.L[0].in_norm, 1, HID, RMS_EPS); - for (int l = 0; l < NL; ++l) { + for (int l = 0; l < NL; ++l) { - LayerW& w = G.L[l]; - const int hd = w.hd, QD = NQH*hd; - const void* rc = w.global ? G.cos_g : G.cos_s; - const void* rs = w.global ? G.sin_g : G.sin_s; - uint64_t* km = (uint64_t*)w.kc; uint64_t* vm = (uint64_t*)w.vc; - size_t mx = km[1], ng = (hd+31)/32; - - void* xn = fresh(HID); cactus_metal_encode_rms_norm(xn, h, w.in_norm, 1, HID, RMS_EPS); - // q/k/v share input xn — batch their CQ4 transforms into one dispatch (non-shared layers). - void *cq=nullptr,*ck=nullptr,*cv=nullptr; bool qkv_batched=false; - if (!w.shared) { - cq=fresh(HID); ck=fresh(HID); cv=fresh(HID); - const CactusQuantMatrix* Wqkv[3]={&w.q,&w.k,&w.v}; void* codes[3]={cq,ck,cv}; - qkv_batched = cactus_metal_encode_transform_batch(xn, Wqkv, 3, codes); - } - void* q = fresh(QD); if(qkv_batched) cactus_metal_encode_gemv_precoded(q, cq, &w.q); else cactus_metal_encode_quant_matmul(q, xn, &w.q); - void* qr = fresh(QD); - if (!g_fuse_nr || !cactus_metal_encode_rms_norm_rope(qr, q, w.q_norm, rc, rs, NQH, hd, RMS_EPS)) { - void* qn = fresh(QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, NQH, hd, RMS_EPS); - cactus_metal_encode_rope(qr, qn, rc, rs, NQH, hd); - } + if (g_flush_layers > 0 && l > 0 && (l % g_flush_layers) == 0) cactus_metal_session_flush(); + LayerW& w = G.L[l]; + const int hd = w.hd, QD = NQH*hd; + const void* rc = w.global ? G.cos_g : G.cos_s; + const void* rs = w.global ? G.sin_g : G.sin_s; + uint64_t* km = (uint64_t*)w.kc; uint64_t* vm = (uint64_t*)w.vc; + size_t mx = km[1], ng = (hd+31)/32; + + void* q = fresh(QD); + void* attn = fresh(QD); + bool fattn = false; + const uint32_t Wn = w.global ? 0u : (uint32_t)(km[1] - km[4] - 1); + const uint32_t Sn = w.global ? 0u : (uint32_t)km[4]; + const uint32_t Rn = (Wn > Sn) ? (Wn - Sn) : 1u; - size_t hist, total; const void *knewp, *vnewp; size_t kv_start = 0, kv_end; - const uint32_t Wn = w.global ? 0u : (uint32_t)(km[1] - km[4] - 1); - const uint32_t Sn = w.global ? 0u : (uint32_t)km[4]; - const uint32_t Rn = (Wn > Sn) ? (Wn - Sn) : 1u; - if (!w.shared) { - void* k = fresh(hd); if(qkv_batched) cactus_metal_encode_gemv_precoded(k, ck, &w.k); else cactus_metal_encode_quant_matmul(k, xn, &w.k); - void* v = fresh(hd); if(qkv_batched) cactus_metal_encode_gemv_precoded(v, cv, &w.v); else cactus_metal_encode_quant_matmul(v, xn, &w.v); - void* kr = fresh(hd); - if (!g_fuse_nr || !cactus_metal_encode_rms_norm_rope(kr, k, w.k_norm, rc, rs, 1, hd, RMS_EPS)) { - void* kn = fresh(hd); cactus_metal_encode_rms_norm(kn, k, w.k_norm, 1, hd, RMS_EPS); - cactus_metal_encode_rope(kr, kn, rc, rs, 1, hd); - } - void* vn = fresh(hd); cactus_metal_encode_rms_norm(vn, v, w.global ? G.vnorm_g : G.vnorm, 1, hd, RMS_EPS); - size_t clen = km[0]; - size_t slot = (!w.global && clen >= (size_t)Wn) ? (size_t)(Sn + ((clen - Sn) % Rn)) : clen; - cactus_metal_encode_kv_append_i8(kr, (char*)w.kc+64, (char*)w.kc+64+mx*hd, - 1, hd, (uint32_t)slot, 32, hd*2, mx*hd, mx*ng*sizeof(float)); - cactus_metal_encode_kv_append_i8(vn, (char*)w.vc+64, (char*)w.vc+64+mx*hd, - 1, hd, (uint32_t)slot, 32, hd*2, mx*hd, mx*ng*sizeof(float)); - km[0] = clen + 1; vm[0] = clen + 1; - if (!w.global && clen >= (size_t)Wn) { - hist = Wn; total = Wn; knewp = nullptr; vnewp = nullptr; kv_end = Wn; + if (!w.shared) { + void* k = fresh(hd); + void* v = fresh(hd); + const CactusQuantMatrix* Wqkv[3]={&w.q,&w.k,&w.v}; + void* yqkv[3]={q,k,v}; + bool qkv_mega = g_mega_qkv && cactus_metal_encode_gemv_mega(yqkv, xn, nullptr, 0.f, Wqkv, 3, nullptr); + if (!qkv_mega) { + void *cq=fresh(HID), *ck=fresh(HID), *cv=fresh(HID); + void* codes[3]={cq,ck,cv}; + bool qkv_batched = cactus_metal_encode_transform_batch(xn, Wqkv, 3, codes); + if (qkv_batched) { + const void* ccodes[3]={cq,ck,cv}; + if (!cactus_metal_encode_gemv_cat(yqkv, ccodes, Wqkv, 3)) { + cactus_metal_group_begin(); + cactus_metal_encode_gemv_precoded(q, cq, &w.q); + cactus_metal_encode_gemv_precoded(k, ck, &w.k); + cactus_metal_encode_gemv_precoded(v, cv, &w.v); + cactus_metal_group_end(); + } + } else { + cactus_metal_encode_quant_matmul(q, xn, &w.q); + cactus_metal_encode_quant_matmul(k, xn, &w.k); + cactus_metal_encode_quant_matmul(v, xn, &w.v); + } + } + size_t clen = km[0]; + bool wrap = (!w.global && clen >= (size_t)Wn); + size_t slot = wrap ? (size_t)(Sn + ((clen - Sn) % Rn)) : clen; + size_t f_kv_end = wrap ? (size_t)Wn : std::min(clen + 1, (size_t)pos_ + 1); + fattn = g_fattn && cactus_metal_encode_attention_fused_i8(attn, q, k, v, + (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, + w.q_norm, w.k_norm, w.global ? G.vnorm_g : G.vnorm, rc, rs, + NQH, (uint32_t)hd, (uint32_t)hd, + 0u, (uint32_t)f_kv_end, (uint32_t)slot, 1u, RMS_EPS, 1.0f, + mx*hd, mx*hd, mx*ng*sizeof(float), mx*ng*sizeof(float)); + if (fattn) { + km[0] = clen + 1; vm[0] = clen + 1; + } else { + void* qr = fresh(QD); + void* kr = fresh(hd); + void* vn = fresh(hd); + bool okq = false, okk = false; + cactus_metal_group_begin(); + if (g_fuse_nr) { + okq = cactus_metal_encode_rms_norm_rope(qr, q, w.q_norm, rc, rs, NQH, hd, RMS_EPS); + okk = cactus_metal_encode_rms_norm_rope(kr, k, w.k_norm, rc, rs, 1, hd, RMS_EPS); + } + cactus_metal_encode_rms_norm(vn, v, w.global ? G.vnorm_g : G.vnorm, 1, hd, RMS_EPS); + cactus_metal_group_end(); + if (!okq) { + void* qn = fresh(QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, NQH, hd, RMS_EPS); + cactus_metal_encode_rope(qr, qn, rc, rs, NQH, hd); + } + if (!okk) { + void* kn = fresh(hd); cactus_metal_encode_rms_norm(kn, k, w.k_norm, 1, hd, RMS_EPS); + cactus_metal_encode_rope(kr, kn, rc, rs, 1, hd); + } + cactus_metal_group_begin(); + cactus_metal_encode_kv_append_i8(kr, (char*)w.kc+64, (char*)w.kc+64+mx*hd, + 1, hd, (uint32_t)slot, 32, hd*2, mx*hd, mx*ng*sizeof(float)); + cactus_metal_encode_kv_append_i8(vn, (char*)w.vc+64, (char*)w.vc+64+mx*hd, + 1, hd, (uint32_t)slot, 32, hd*2, mx*hd, mx*ng*sizeof(float)); + cactus_metal_group_end(); + km[0] = clen + 1; vm[0] = clen + 1; + size_t hist, total, kv_end; const void *knewp, *vnewp; + if (wrap) { + hist = Wn; total = Wn; knewp = nullptr; vnewp = nullptr; kv_end = Wn; + } else { + hist = clen; total = clen + 1; knewp = kr; vnewp = vn; + kv_end = std::min(total, (size_t)pos_ + 1); + } + cactus_metal_encode_attention_i8(attn, qr, knewp, vnewp, + (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, + NQH, 1, hd, hd, (uint32_t)hist, (uint32_t)total, 0u, (uint32_t)kv_end, 1.0f, + hist*hd, hist*hd, hist*ng*sizeof(float), hist*ng*sizeof(float)); + } } else { - hist = clen; total = clen + 1; knewp = kr; vnewp = vn; - kv_end = std::min(total, (size_t)pos + 1); + const CactusQuantMatrix* Wq1[1]={&w.q}; + void* yq1[1]={q}; + if (!g_mega_qkv || !cactus_metal_encode_gemv_mega(yq1, xn, nullptr, 0.f, Wq1, 1, nullptr)) { + cactus_metal_encode_quant_matmul(q, xn, &w.q); + } + size_t clen = km[0]; + size_t s_kv_end = (!w.global && clen > (size_t)Wn) ? (size_t)Wn : std::min(clen, (size_t)pos_ + 1); + fattn = g_fattn && cactus_metal_encode_attention_fused_i8(attn, q, nullptr, nullptr, + (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, + w.q_norm, nullptr, nullptr, rc, rs, + NQH, (uint32_t)hd, (uint32_t)hd, + 0u, (uint32_t)s_kv_end, 0u, 0u, RMS_EPS, 1.0f, + mx*hd, mx*hd, mx*ng*sizeof(float), mx*ng*sizeof(float)); + if (!fattn) { + void* qr = fresh(QD); + if (!g_fuse_nr || !cactus_metal_encode_rms_norm_rope(qr, q, w.q_norm, rc, rs, NQH, hd, RMS_EPS)) { + void* qn = fresh(QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, NQH, hd, RMS_EPS); + cactus_metal_encode_rope(qr, qn, rc, rs, NQH, hd); + } + size_t hist, total, kv_end; const void *knewp, *vnewp; + if (!w.global && clen > (size_t)Wn) { + hist = Wn; total = Wn; knewp = nullptr; vnewp = nullptr; kv_end = Wn; + } else { + hist = clen; total = clen; knewp = qr; vnewp = qr; + kv_end = std::min(clen, (size_t)pos_ + 1); + } + cactus_metal_encode_attention_i8(attn, qr, knewp, vnewp, + (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, + NQH, 1, hd, hd, (uint32_t)hist, (uint32_t)total, 0u, (uint32_t)kv_end, 1.0f, + hist*hd, hist*hd, hist*ng*sizeof(float), hist*ng*sizeof(float)); + } } - } else { - size_t clen = km[0]; - if (!w.global && clen > (size_t)Wn) { - hist = Wn; total = Wn; knewp = nullptr; vnewp = nullptr; kv_end = Wn; + void* o = fresh(HID); + { + const CactusQuantMatrix* Wo[1]={&w.o}; void* yo[1]={o}; + if (!g_mega_o || !cactus_metal_encode_gemv_mega(yo, attn, nullptr, 0.f, Wo, 1, nullptr)) + cactus_metal_encode_quant_matmul(o, attn, &w.o); + } + void* h1 = fresh(HID); + void* xn2 = fresh(HID); + if (!cactus_metal_encode_rms_norm_add_rms(h1, xn2, o, w.post_attn, h, w.pre_ffn, 1, HID, RMS_EPS, 1.0f)) { + cactus_metal_encode_rms_norm_add(h1, o, w.post_attn, h, 1, HID, RMS_EPS); + cactus_metal_encode_rms_norm(xn2, h1, w.pre_ffn, 1, HID, RMS_EPS); + } + h = h1; + + const int M = w.mlp; + void* gate= fresh(M); + void* up = fresh(M); + const CactusQuantMatrix* Wgu[2]={&w.gate,&w.up}; + { + void* ygu[2]={gate,up}; + if (!g_mega_gu || !cactus_metal_encode_gemv_mega(ygu, xn2, nullptr, 0.f, Wgu, 2, nullptr)) { + void *cg=fresh(HID), *cu=fresh(HID); void* cgu[2]={cg,cu}; + bool gu_batched = cactus_metal_encode_transform_batch(xn2, Wgu, 2, cgu); + if (gu_batched) { + const void* gcodes[2]={cg,cu}; + if (!cactus_metal_encode_gemv_cat(ygu, gcodes, Wgu, 2)) { + cactus_metal_group_begin(); + cactus_metal_encode_gemv_precoded(gate, cg, &w.gate); + cactus_metal_encode_gemv_precoded(up, cu, &w.up); + cactus_metal_group_end(); + } + } else { + cactus_metal_encode_quant_matmul(gate, xn2, &w.gate); + cactus_metal_encode_quant_matmul(up, xn2, &w.up); + } + } + } + void* mo = fresh(HID); + { + const CactusQuantMatrix* Wd[1]={&w.down}; void* ymo[1]={mo}; + void* gcode = fresh(M); + if (g_mega_down && cactus_metal_encode_gemv_mega_ex(ymo, gate, nullptr, 0.f, Wd, 1, up, true, GATE_SCALE)) { + } else if (g_mega_dswt && cactus_metal_encode_swiglu_transform(gcode, gate, up, &w.down, GATE_SCALE) + && cactus_metal_encode_gemv_precoded(mo, gcode, &w.down)) { + } else { + void* g3 = fresh(M); cactus_metal_encode_swiglu(g3, gate, up, M, GATE_SCALE); + cactus_metal_encode_quant_matmul(mo, g3, &w.down); + } + } + void* h2 = fresh(HID); cactus_metal_encode_rms_norm_add(h2, mo, w.post_ffn, h, 1, HID, RMS_EPS); h = h2; + + void* pm = fresh(PLE_DIM); + void* pu = fresh(HID); + { + const void* pls = (const char*)pleBase + (size_t)l*PLE_DIM*2; + const CactusQuantMatrix* Wpd[1]={&w.ple_down}; void* ypd[1]={pm}; + const CactusQuantMatrix* Wpu[1]={&w.ple_up}; void* ypu[1]={pu}; + if (g_mega_ple && cactus_metal_encode_gemv_mega(ypd, h, nullptr, 0.f, Wpd, 1, pls) + && cactus_metal_encode_gemv_mega(ypu, pm, nullptr, 0.f, Wpu, 1, nullptr)) { + } else { + void* ps = fresh(PLE_DIM); cactus_metal_encode_quant_matmul(ps, h, &w.ple_down); + cactus_metal_encode_swiglu(pm, ps, pls, PLE_DIM, 1.0f); + cactus_metal_encode_quant_matmul(pu, pm, &w.ple_up); + } + } + float ls = (float)(*(const __fp16*)w.scalar); + void* h3 = fresh(HID); + void* xnn = fresh(HID); + const void* next_w = (l+1 < NL) ? G.L[l+1].in_norm : G.final_norm; + if (cactus_metal_encode_rms_norm_add_rms(h3, xnn, pu, w.post_ple, h, next_w, 1, HID, RMS_EPS, ls)) { + h = h3; xn = xnn; + } else if (g_fuse_nr && cactus_metal_encode_rms_norm_add_scale(h3, pu, w.post_ple, h, 1, HID, RMS_EPS, ls)) { + cactus_metal_encode_rms_norm(xnn, h3, next_w, 1, HID, RMS_EPS); + h = h3; xn = xnn; } else { - hist = clen; total = clen; knewp = qr; vnewp = qr; - kv_end = std::min(clen, (size_t)pos + 1); + cactus_metal_encode_rms_norm_add(h3, pu, w.post_ple, h, 1, HID, RMS_EPS); + void* h4 = fresh(HID); cactus_metal_encode_scalar(2, h4, h3, HID, ls); + cactus_metal_encode_rms_norm(xnn, h4, next_w, 1, HID, RMS_EPS); + h = h4; xn = xnn; } } - void* attn = fresh(QD); - cactus_metal_encode_attention_i8(attn, qr, knewp, vnewp, - (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, - NQH, 1, hd, hd, (uint32_t)hist, (uint32_t)total, (uint32_t)kv_start, (uint32_t)kv_end, 1.0f, - hist*hd, hist*hd, hist*ng*sizeof(float), hist*ng*sizeof(float)); - void* o = fresh(HID); cactus_metal_encode_quant_matmul(o, attn, &w.o); - void* h1 = fresh(HID); cactus_metal_encode_rms_norm_add(h1, o, w.post_attn, h, 1, HID, RMS_EPS); h = h1; - - const int M = w.mlp; - void* xn2 = fresh(HID); cactus_metal_encode_rms_norm(xn2, h, w.pre_ffn, 1, HID, RMS_EPS); - // gate/up share input xn2 — batch their transforms. - void *cg=fresh(HID), *cu=fresh(HID); const CactusQuantMatrix* Wgu[2]={&w.gate,&w.up}; void* cgu[2]={cg,cu}; - bool gu_batched = cactus_metal_encode_transform_batch(xn2, Wgu, 2, cgu); - void* gate= fresh(M); if(gu_batched) cactus_metal_encode_gemv_precoded(gate, cg, &w.gate); else cactus_metal_encode_quant_matmul(gate, xn2, &w.gate); - void* up = fresh(M); if(gu_batched) cactus_metal_encode_gemv_precoded(up, cu, &w.up); else cactus_metal_encode_quant_matmul(up, xn2, &w.up); - void* g3 = fresh(M); cactus_metal_encode_swiglu(g3, gate, up, M, GATE_SCALE); - void* mo = fresh(HID); cactus_metal_encode_quant_matmul(mo, g3, &w.down); - void* h2 = fresh(HID); cactus_metal_encode_rms_norm_add(h2, mo, w.post_ffn, h, 1, HID, RMS_EPS); h = h2; - - void* ps = fresh(PLE_DIM); cactus_metal_encode_quant_matmul(ps, h, &w.ple_down); - void* pm = fresh(PLE_DIM); cactus_metal_encode_swiglu(pm, ps, (const char*)pleBase + (size_t)l*PLE_DIM*2, PLE_DIM, 1.0f); - void* pu = fresh(HID); cactus_metal_encode_quant_matmul(pu, pm, &w.ple_up); - // fold the per-layer residual scalar into the post-PLE norm+residual (one pass instead of two) - float ls = (float)(*(const __fp16*)w.scalar); - void* h3 = fresh(HID); - if (g_fuse_nr && cactus_metal_encode_rms_norm_add_scale(h3, pu, w.post_ple, h, 1, HID, RMS_EPS, ls)) { - h = h3; - } else { - cactus_metal_encode_rms_norm_add(h3, pu, w.post_ple, h, 1, HID, RMS_EPS); - void* h4 = fresh(HID); cactus_metal_encode_scalar(2, h4, h3, HID, ls); h = h4; - } - } - size_t V = B(3860).total_size; - void* fn = fresh(HID); cactus_metal_encode_rms_norm(fn, h, G.final_norm, 1, HID, RMS_EPS); - void* code = fresh(HID); - void* lg = fresh(V); - if (cactus_metal_encode_quant_matmul_ortho(lg, fn, code, &G.lm_head)) { - cactus_metal_encode_scalar(3, lg, lg, V, SOFTCAP); - cactus_metal_encode_unary(1, lg, lg, V); - cactus_metal_encode_scalar(2, lg, lg, V, SOFTCAP); - B(3860).set_external(lg); - if (G.argmax_buf && cactus_metal_encode_argmax(lg, (uint32_t)V, G.argmax_buf)) { - g_gpu_argmax_buf = (const float*)G.argmax_buf; - g_gpu_argmax_valid = true; + size_t V = B(3860).total_size; + void* fn = xn; + void* code = fresh(HID); + void* lg = fresh(V); + *lg_out = lg; + if (cactus_metal_encode_quant_matmul_ortho(lg, fn, code, &G.lm_head)) { + if (!cactus_metal_encode_softcap(lg, lg, V, SOFTCAP)) { + cactus_metal_encode_scalar(3, lg, lg, V, SOFTCAP); + cactus_metal_encode_unary(1, lg, lg, V); + cactus_metal_encode_scalar(2, lg, lg, V, SOFTCAP); + } + bool adjusted = false; + if (sampling_needs_adjust()) { + adjusted = cactus_metal_encode_adjust_logits(lg, V, + g_samp.recent.data(), (uint32_t)g_samp.recent.size(), + g_samp.bias_ids.data(), g_samp.bias_vals.data(), (uint32_t)g_samp.bias_ids.size(), + g_samp.suppressed, g_samp.rep_penalty); + if (!adjusted) return 0; + } + g_step_adjusted = adjusted; + cactus_metal_encode_argmax(lg, (uint32_t)V, G.amax); + return 1; } - } else { cactus_metal_session_sync(); cactus_quant_matmul(&G.lm_head, (const __fp16*)fn, 1u, (__fp16*)lg); __fp16* L=(__fp16*)lg; for (size_t i=0;i1 prefill: mirrors execute_gpu_fused (decode) for a chunk of M tokens, keeping every -// intermediate on-chip instead of round-tripping the generic graph executor. Weight/norm node -// indices are IDENTICAL to the decode graph (verified). Falls back (returns false) on any KV wrap -// or unsupported shape so the generic path stays authoritative for the hard cases. bool CactusGraph::execute_gpu_fused_prefill(uint32_t M) { if (!cactus_metal_available() || M <= 1) return false; const size_t n = nodes_.size(); @@ -376,7 +577,6 @@ bool CactusGraph::execute_gpu_fused_prefill(uint32_t M) { if (nodes_[i]->op_type == OpType::KV_CACHE_STATE) cidx.push_back(i); if (cidx.size() < 30) return false; for (int l = 0; l < 15; ++l) { GP.kc_idx[l]=cidx[2*l]; GP.vc_idx[l]=cidx[2*l+1]; } - // per-layer attention params from the graph's ATTENTION_CACHED nodes (in layer order) int ai = 0; for (size_t i = 0; i < n && ai < NL; ++i) { if (nodes_[i]->op_type != OpType::ATTENTION_CACHED) continue; @@ -386,21 +586,22 @@ bool CactusGraph::execute_gpu_fused_prefill(uint32_t M) { ai++; } if (ai < NL) return false; + GP.owner = this; GP.init = true; } + if (GP.owner != this) { + GP.init = false; + return false; + } for (int l = 0; l < 15; ++l) { GP.L[l].kc = P(GP.kc_idx[l]); GP.L[l].vc = P(GP.vc_idx[l]); } for (int l = 15; l < NL; ++l) { int src = GP.L[l].global ? 14 : 13; GP.L[l].kc = GP.L[src].kc; GP.L[l].vc = GP.L[src].vc; } for (int l = 0; l < NL; ++l) if (!GP.L[l].kc || !GP.L[l].vc) return false; - // Chunk start position = tokens already in the cache (before this chunk's append). RoPE rotates - // this chunk's tokens at absolute positions pos0..pos0+M-1. Derive from the KV header (robust to - // however the graph's position input is populated). int pos0 = (int)((const uint64_t*)GP.L[0].kc)[0]; - // Safe first version: bail (generic handles it) if any owned KV cache would wrap this chunk. for (int l = 0; l < 15; ++l) { const uint64_t* km = (const uint64_t*)GP.L[l].kc; - if (km[0] + M > km[1]) return false; // would exceed cache capacity -> ring/sliding path + if (km[0] + M > km[1]) return false; } GP.arena.reset(); @@ -409,7 +610,6 @@ bool CactusGraph::execute_gpu_fused_prefill(uint32_t M) { cactus_metal_set_active(true); auto bail = [&](){ cactus_metal_set_active(false); cactus_metal_session_end(); return false; }; - // gather per-token cos/sin for the M positions (local hd=256, global hd=512) void* csL = fresh((size_t)M*256); void* snL = fresh((size_t)M*256); void* csG = fresh((size_t)M*512); void* snG = fresh((size_t)M*512); for (uint32_t m = 0; m < M; ++m) { @@ -495,7 +695,6 @@ bool CactusGraph::execute_gpu_fused_prefill(uint32_t M) { void* mo = fresh((size_t)M*HID); if(!cactus_metal_encode_quant_matmul_m(mo, g3, &w.down, M)) return bail(); void* h2 = fresh((size_t)M*HID); cactus_metal_encode_rms_norm_add(h2, mo, w.post_ffn, h, M, HID, RMS_EPS); h = h2; - // PLE: gather ple[*, l, *] (strided by NL*PLE_DIM) into a contiguous [M, PLE_DIM] slice void* ps = fresh((size_t)M*PLE_DIM); if(!cactus_metal_encode_quant_matmul_m(ps, h, &w.ple_down, M)) return bail(); void* pled= fresh((size_t)M*PLE_DIM); { uint32_t osh[2]={M,(uint32_t)PLE_DIM}; uint32_t sst[2]={(uint32_t)(NL*PLE_DIM),1}; @@ -509,18 +708,29 @@ bool CactusGraph::execute_gpu_fused_prefill(uint32_t M) { void* h4 = fresh((size_t)M*HID); cactus_metal_encode_scalar(2, h4, h, (size_t)M*HID, ls); h = h4; } - // logits for the LAST token only (prepare_decode needs just that one) const void* hlast = (const char*)h + (size_t)(M-1)*HID*2; size_t V = B(n-1).total_size; void* fn = fresh(HID); cactus_metal_encode_rms_norm(fn, hlast, GP.final_norm, 1, HID, RMS_EPS); void* code = fresh(HID); void* lg = fresh(V); if (!cactus_metal_encode_quant_matmul_ortho(lg, fn, code, &GP.lm_head)) return bail(); - cactus_metal_encode_scalar(3, lg, lg, V, SOFTCAP); - cactus_metal_encode_unary(1, lg, lg, V); - cactus_metal_encode_scalar(2, lg, lg, V, SOFTCAP); + if (!cactus_metal_encode_softcap(lg, lg, V, SOFTCAP)) { + cactus_metal_encode_scalar(3, lg, lg, V, SOFTCAP); + cactus_metal_encode_unary(1, lg, lg, V); + cactus_metal_encode_scalar(2, lg, lg, V, SOFTCAP); + } B(n-1).set_external(lg); cactus_metal_set_active(false); cactus_metal_session_end(); return true; } + +void cactus_graph_on_destroy(const void* graph) { + if (G.init && G.owner == graph) G.init = false; + if (GP.init && GP.owner == graph) GP.init = false; + cactus_metal_invalidate_host_wraps(); +} + +CactusGraph::~CactusGraph() { + cactus_graph_on_destroy(this); +} diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 47d81ba2c..4550784c8 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -24,11 +24,17 @@ bool cactus_metal_active_mode(); bool cactus_metal_concurrent(); void cactus_metal_barrier(); +void cactus_metal_session_set_concurrent(bool on); +void cactus_metal_group_begin(); +void cactus_metal_group_end(); + void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, uint32_t M, __fp16* C); void cactus_metal_session_begin(); void cactus_metal_session_sync(); +void cactus_metal_session_flush(); +void cactus_metal_invalidate_host_wraps(); void cactus_metal_session_end(); void* cactus_metal_alloc_shared(size_t bytes); @@ -50,18 +56,35 @@ bool cactus_metal_encode_rms_norm_rope(void* out, const void* x, const void* wei uint32_t heads, uint32_t head_dim, float eps); bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* weight, const void* res, size_t rows, size_t dim, float eps); +bool cactus_metal_encode_rms_norm_add_rms(void* h_out, void* xn_out, const void* in, const void* w1, + const void* res, const void* w2, + size_t rows, size_t dim, float eps, float out_scale); bool cactus_metal_encode_rms_norm_add_scale(void* out, const void* in, const void* weight, const void* res, size_t rows, size_t dim, float eps, float out_scale); bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3); +bool cactus_metal_encode_softcap(void* out, const void* in, size_t n, float cap); +bool cactus_metal_encode_adjust_logits(void* logits, size_t vocab, + const uint32_t* recent, uint32_t n_recent, + const uint32_t* bias_ids, const float* bias_vals, uint32_t n_bias, + int64_t suppressed, float penalty); bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_prec, size_t n); bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W); bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M); bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* const* Ws, int B, void* const* codes); bool cactus_metal_encode_gemv_precoded(void* out, const void* code, const CactusQuantMatrix* W); +bool cactus_metal_encode_gemv_mega(void* const* outs, const void* x, const void* normw, float eps, + const CactusQuantMatrix* const* Ws, int B, const void* osw); +bool cactus_metal_encode_gemv_mega_ex(void* const* outs, const void* x, const void* normw, float eps, + const CactusQuantMatrix* const* Ws, int B, const void* osw, + bool pswi, float pscale); +bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, + const CactusQuantMatrix* const* Ws, int B); +bool cactus_metal_encode_swiglu_transform(void* code, const void* gate, const void* up, + const CactusQuantMatrix* W, float scale); bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W); bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* code, const CactusQuantMatrix* W); -bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W); +bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W, float scale); bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const CactusQuantMatrix* W); bool cactus_metal_encode_embedding_ortho_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M); bool cactus_metal_encode_embedding_hadamard_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M); @@ -74,6 +97,15 @@ bool cactus_metal_encode_attention_i8( uint32_t history_len, uint32_t total_keys, uint32_t kv_start, uint32_t kv_end, float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes); +bool cactus_metal_encode_attention_fused_i8( + void* out, const void* q, const void* kraw, const void* vraw, + void* kc, void* vc, void* ks, void* vs, + const void* qw, const void* kw, const void* vw, const void* cs, const void* sn, + uint32_t nqh, uint32_t hd, uint32_t vhd, + uint32_t kv_start, uint32_t kv_end, uint32_t slot, uint32_t has_new, + float eps, float scale, + size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes); + bool cactus_metal_encode_attention_i8_prefill( void* out, const void* q, const void* knew, const void* vnew, const void* kc, const void* vc, const void* ks, const void* vs, diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index c8edefaac..4953770d6 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -2,6 +2,11 @@ using namespace metal; #define ROWS 8u +inline float gelu_tanh(float x) { + float c = 0.7978845608028654f * (x + 0.044715f*x*x*x); + return 0.5f * x * (1.0f + precise::tanh(c)); +} + kernel void cq4_transform( device const half* x [[buffer(0)]], device const half* recip [[buffer(1)]], @@ -62,10 +67,6 @@ kernel void cq4_transform_simd( code[b+2]=(half)zmem[perm[k+2]]; code[b+3]=(half)zmem[perm[k+3]]; } -// Batched CQ4 transform: one dispatch transforms the SAME input `x` for up to 3 weights that share the -// input (q/k/v use xn; gate/up use xn2 — verified `recip` is identical, only lsign/rsign/perm differ per -// weight). Grid = ng*B threadgroups; threadgroup (g + ng*b) transforms group g with weight b's params into -// code[b]. Replaces B separate transform dispatches + removes the g_code_buf WAR serialization. kernel void cq4_transform_batch( device const half* x [[buffer(0)]], device const half* recip [[buffer(1)]], @@ -148,10 +149,6 @@ kernel void cq4_gemv( if (lane==0) y[n]=(half)acc; } -// Multi-row CQ4 gemv (M=1 decode): each simdgroup computes CQ4_NR output rows instead of 1. The shared -// activation `code` is read once per K-step and reused across the NR rows, while the NR weight streams are -// read interleaved (unrolled) — more independent memory requests in flight to hide M=1 latency (llama -// mul_mv `nr0` technique). ROWS*CQ4_NR rows per threadgroup; N is a multiple of 32 for all matmuls so no tail. #define CQ4_NR 2u kernel void cq4_gemv_mr( device const half* code [[buffer(0)]], @@ -200,6 +197,230 @@ kernel void cq4_gemv_mr( } } +struct MegaU { uint K, ng, nw, N0, N1, N2, flags; float eps; float pscale; }; +kernel void cq4_gemv_mega( + device const half* x [[buffer(0)]], + device const half* normw [[buffer(1)]], + device const half* recip [[buffer(2)]], + device const char* lsign0 [[buffer(3)]], device const char* lsign1 [[buffer(4)]], device const char* lsign2 [[buffer(5)]], + device const char* rsign0 [[buffer(6)]], device const char* rsign1 [[buffer(7)]], device const char* rsign2 [[buffer(8)]], + device const uint* perm0 [[buffer(9)]], device const uint* perm1 [[buffer(10)]], device const uint* perm2 [[buffer(11)]], + device const half* cbook0 [[buffer(12)]], device const half* cbook1 [[buffer(13)]], device const half* cbook2 [[buffer(14)]], + device const half* norms0 [[buffer(15)]], device const half* norms1 [[buffer(16)]], device const half* norms2 [[buffer(17)]], + device const uchar* packed0 [[buffer(18)]], device const uchar* packed1[[buffer(19)]], device const uchar* packed2[[buffer(20)]], + device half* y0 [[buffer(21)]], device half* y1 [[buffer(22)]], device half* y2 [[buffer(23)]], + device const half* osw [[buffer(24)]], + constant MegaU& U [[buffer(25)]], + uint tg [[threadgroup_position_in_grid]], + uint sg [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint t [[thread_position_in_threadgroup]], + threadgroup half* coded [[threadgroup(0)]], + threadgroup float* zwave [[threadgroup(1)]], + threadgroup float* red [[threadgroup(2)]]) +{ + const uint K = U.K, ng = U.ng; + const bool do_norm = (U.flags & 1u) != 0u; + const bool do_oswi = (U.flags & 2u) != 0u; + const bool do_pswi = (U.flags & 4u) != 0u; + + uint n0 = tg*16u; + uint b, nbase; + if (n0 < U.N0) { b = 0u; nbase = 0u; } + else if (n0 < U.N0 + U.N1) { b = 1u; nbase = U.N0; } + else { b = 2u; nbase = U.N0 + U.N1; } + device const char* lsign = (b==0u)?lsign0:((b==1u)?lsign1:lsign2); + device const char* rsign = (b==0u)?rsign0:((b==1u)?rsign1:rsign2); + device const uint* perm = (b==0u)?perm0 :((b==1u)?perm1 :perm2 ); + device const half* cbook = (b==0u)?cbook0:((b==1u)?cbook1:cbook2); + device const half* norms = (b==0u)?norms0:((b==1u)?norms1:norms2); + device const uchar* packed= (b==0u)?packed0:((b==1u)?packed1:packed2); + device half* y = (b==0u)?y0 :((b==1u)?y1 :y2 ); + + threadgroup float* cb = red; + + float inv = 1.0f; + if (do_norm) { + float partial = 0; + for (uint i=t; i0u; s>>=1){ if (t>4)&0xF] + (float)c0.z*cb[(w[0]>>8)&0xF] + (float)c0.w*cb[(w[0]>>12)&0xF] + + (float)c1.x*cb[w[1]&0xF] + (float)c1.y*cb[(w[1]>>4)&0xF] + (float)c1.z*cb[(w[1]>>8)&0xF] + (float)c1.w*cb[(w[1]>>12)&0xF] + + (float)c2.x*cb[w[2]&0xF] + (float)c2.y*cb[(w[2]>>4)&0xF] + (float)c2.z*cb[(w[2]>>8)&0xF] + (float)c2.w*cb[(w[2]>>12)&0xF] + + (float)c3.x*cb[w[3]&0xF] + (float)c3.y*cb[(w[3]>>4)&0xF] + (float)c3.z*cb[(w[3]>>8)&0xF] + (float)c3.w*cb[(w[3]>>12)&0xF]; + acc[r] += (float)norms[(size_t)n*ng+g]*p; + } + } + #pragma clang loop unroll(full) + for (uint r=0u;r<2u;r++){ + float a = simd_sum(acc[r]); + if (lane==0u){ + uint n = nloc0+r; + if (do_oswi) { + float gg = gelu_tanh((float)(half)a) * (float)osw[n]; + y[n] = (half)clamp(gg,-65504.0f,65504.0f); + } else { + y[n] = (half)a; + } + } + } +} + +kernel void cq4_swiglu_transform( + device const half* gate [[buffer(0)]], + device const half* up [[buffer(1)]], + device const half* recip [[buffer(2)]], + device const char* lsign [[buffer(3)]], + device const char* rsign [[buffer(4)]], + device const uint* perm [[buffer(5)]], + device half* code [[buffer(6)]], + constant float& scale [[buffer(7)]], + uint g [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + threadgroup float* zmem [[threadgroup(0)]]) +{ + uint b = g*128u + lane*4u; + uint k = lane*4u; + float x0,x1,x2,x3; + { + float g0 = gelu_tanh((float)gate[b+0]) * scale * (float)up[b+0]; + float g1 = gelu_tanh((float)gate[b+1]) * scale * (float)up[b+1]; + float g2 = gelu_tanh((float)gate[b+2]) * scale * (float)up[b+2]; + float g3 = gelu_tanh((float)gate[b+3]) * scale * (float)up[b+3]; + x0=(float)(half)clamp(g0,-65504.0f,65504.0f)*(float)recip[b+0]*(float)lsign[k+0]; + x1=(float)(half)clamp(g1,-65504.0f,65504.0f)*(float)recip[b+1]*(float)lsign[k+1]; + x2=(float)(half)clamp(g2,-65504.0f,65504.0f)*(float)recip[b+2]*(float)lsign[k+2]; + x3=(float)(half)clamp(g3,-65504.0f,65504.0f)*(float)recip[b+3]*(float)lsign[k+3]; + } + float a0=x0+x1,a1=x0-x1,a2=x2+x3,a3=x2-x3; + x0=a0+a2; x1=a1+a3; x2=a0-a2; x3=a1-a3; + #pragma clang loop unroll(full) + for (uint d=1u; d<=16u; d<<=1){ + bool hi=(lane&d)!=0u; + float p0=simd_shuffle_xor(x0,d),p1=simd_shuffle_xor(x1,d),p2=simd_shuffle_xor(x2,d),p3=simd_shuffle_xor(x3,d); + x0=hi?p0-x0:x0+p0; x1=hi?p1-x1:x1+p1; x2=hi?p2-x2:x2+p2; x3=hi?p3-x3:x3+p3; + } + float s=rsqrt(128.0f); + zmem[k+0]=x0*s*(float)rsign[k+0]; zmem[k+1]=x1*s*(float)rsign[k+1]; + zmem[k+2]=x2*s*(float)rsign[k+2]; zmem[k+3]=x3*s*(float)rsign[k+3]; + threadgroup_barrier(mem_flags::mem_threadgroup); + code[b+0]=(half)zmem[perm[k+0]]; code[b+1]=(half)zmem[perm[k+1]]; + code[b+2]=(half)zmem[perm[k+2]]; code[b+3]=(half)zmem[perm[k+3]]; +} + +struct CatU { uint ng, N0, N1, N2; }; +kernel void cq4_gemv_cat( + device const half* code0 [[buffer(0)]], device const half* code1 [[buffer(1)]], device const half* code2 [[buffer(2)]], + device const uchar* packed0[[buffer(3)]], device const uchar* packed1[[buffer(4)]], device const uchar* packed2[[buffer(5)]], + device const half* cbook0 [[buffer(6)]], device const half* cbook1 [[buffer(7)]], device const half* cbook2 [[buffer(8)]], + device const half* norms0 [[buffer(9)]], device const half* norms1 [[buffer(10)]], device const half* norms2 [[buffer(11)]], + device half* y0 [[buffer(12)]], device half* y1 [[buffer(13)]], device half* y2 [[buffer(14)]], + constant CatU& U [[buffer(15)]], + uint tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]]) +{ + uint n0 = tg*16u; + uint b, nbase; + if (n0 < U.N0) { b = 0u; nbase = 0u; } + else if (n0 < U.N0 + U.N1) { b = 1u; nbase = U.N0; } + else { b = 2u; nbase = U.N0 + U.N1; } + device const half* code = (b==0u)?code0 :((b==1u)?code1 :code2 ); + device const uchar* packed= (b==0u)?packed0:((b==1u)?packed1:packed2); + device const half* cbook = (b==0u)?cbook0:((b==1u)?cbook1:cbook2); + device const half* norms = (b==0u)?norms0:((b==1u)?norms1:norms2); + device half* y = (b==0u)?y0 :((b==1u)?y1 :y2 ); + threadgroup float cb[16]; + if (tl<16) cb[tl]=(float)cbook[tl]; + threadgroup_barrier(mem_flags::mem_threadgroup); + uint ng = U.ng; + uint K = ng*128u; + uint nl0 = (n0 - nbase) + sgid*2u; + float acc[2]; acc[0]=0; acc[1]=0; + for (uint base=lane*16u; base>4)&0xF] + (float)c0.z*cb[(w[0]>>8)&0xF] + (float)c0.w*cb[(w[0]>>12)&0xF] + + (float)c1.x*cb[w[1]&0xF] + (float)c1.y*cb[(w[1]>>4)&0xF] + (float)c1.z*cb[(w[1]>>8)&0xF] + (float)c1.w*cb[(w[1]>>12)&0xF] + + (float)c2.x*cb[w[2]&0xF] + (float)c2.y*cb[(w[2]>>4)&0xF] + (float)c2.z*cb[(w[2]>>8)&0xF] + (float)c2.w*cb[(w[2]>>12)&0xF] + + (float)c3.x*cb[w[3]&0xF] + (float)c3.y*cb[(w[3]>>4)&0xF] + (float)c3.z*cb[(w[3]>>8)&0xF] + (float)c3.w*cb[(w[3]>>12)&0xF]; + acc[r] += (float)norms[(size_t)n*ng+g]*p; + } + } + #pragma clang loop unroll(full) + for (uint r=0;r<2;r++){ + float a = simd_sum(acc[r]); + if (lane==0) y[nl0+r]=(half)a; + } +} + kernel void cq4_transform_m( device const half* x [[buffer(0)]], device const half* recip [[buffer(1)]], @@ -293,21 +514,16 @@ kernel void cq4_gemm_mma( uint tl [[thread_index_in_threadgroup]], uint sg [[simdgroup_index_in_threadgroup]]) { - // llama.cpp-style SG_MAT gemm (ported from ggml-metal kernel_mul_mm, non-tensor path): shared tiles - // are stored as CONTIGUOUS 8x8 blocks so simdgroup_load is conflict-free (vs row-major ldA=40 strided). - // Weight is dequantized (CQ4 codebook) into sa; activation (code) into sb. The fill also transposes - // weight [N,K]->[K,N] via the ly/lx block indexing. Tile 64 N-cols x 32 M-rows x 32 K, 128 threads - // (4 simdgroups), mc[8] accumulators. ~2x faster than the old 64x32 row-major kernel (dense microbench). - threadgroup float pool[2048]; // 8KB: sa+sb (6KB) during K-loop, Cs (8KB) at store + threadgroup float pool[2048]; threadgroup float cb[16]; - threadgroup half* sa = (threadgroup half*)pool; // weight 64Nx32K SG_MAT = 2048 halves (4KB) - threadgroup half* sb = sa + 2048u; // activ. 32Mx32K SG_MAT = 1024 halves (2KB) - threadgroup float* Cs = pool; // 32M x 64N float (8KB), aliases sa+sb at store + threadgroup half* sa = (threadgroup half*)pool; + threadgroup half* sb = sa + 2048u; + threadgroup float* Cs = pool; if (tl<16u) cb[tl]=(float)codebook[tl]; const uint NK=32u, NL0=2u, NL1=4u; uint K = num_groups*gs; - uint r0 = tg.y*64u; // N tile (output features) - uint r1 = tg.x*32u; // M tile (tokens) + uint r0 = tg.y*64u; + uint r1 = tg.x*32u; uint lr0 = tl/NL0, lr1 = tl/NL1, il0 = tl%NL0; uint nrow = r0+lr0; uint iy = 8u*(tl%NL1); @@ -318,7 +534,7 @@ kernel void cq4_gemm_mma( threadgroup_barrier(mem_flags::mem_threadgroup); for (uint loop_k=0; loop_k>1u), n_sg = 32u*(sg&1u); for (uint i=0;i<8u;i++) simdgroup_store(mc[i], &Cs[(m_sg+8u*(i/4u))*64u + n_sg+8u*(i%4u)], 64u); @@ -353,9 +569,6 @@ kernel void cq4_gemm_mma( } } -// Dequantize a CQ4 weight [N,K] to dense fp16 (W[n][k] = norms[n][g] * codebook[nibble]) so prefill can -// run a plain fp16 SG_MAT gemm instead of dequantizing in the gemm fill (the ~33% CQ4 tax). One thread per -// (row,group) writes gs values. Done ONCE per weight (cached), so the cost amortizes across all chunks. kernel void cq4_dequant_f16( device const uchar* packed [[buffer(0)]], device const half* codebook [[buffer(1)]], @@ -380,9 +593,6 @@ kernel void cq4_dequant_f16( } } -// Dense fp16 SG_MAT gemm: out[M,N] = code[M,K] @ dequant(W)[N,K]^T, reading pre-dequantized wf16 (no -// codebook LUT in the fill). Same tile/schedule as cq4_gemm_mma (64N x 32M x 32K, 128 threads) but the -// weight fill is a direct load, so it hits the dense microbench rate (~5257 vs ~3522 GFLOP/s dequant-in-fill). kernel void cq4_gemm_dense_f16( device const half* code [[buffer(0)]], device const half* wf16 [[buffer(1)]], @@ -447,6 +657,55 @@ kernel void lmhead_rotate(device const half* act [[buffer(0)]], device const hal code[i] = (half)acc; } +kernel void lmhead_rotate_wide(device const half* act [[buffer(0)]], device const half* recip [[buffer(1)]], + device const half* rotation [[buffer(2)]], device half* code [[buffer(3)]], + constant uint& K [[buffer(4)]], + uint tg [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + threadgroup float* red [[threadgroup(0)]]) { + uint c = t & 31u, r = t >> 5u; + uint j = tg*32u + c; + float acc = 0; + if (j < K) for (uint k=r; k>1]; uint idx=(i&1u)?(uint)(b>>4):(uint)(b&0xF); dq[i]=(float)codebook[idx]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + float nrm = (float)norms[row]; + uint j = tg*8u + sg; + if (j >= K) return; + device const half* rrow = rotation + (size_t)j*K; + float acc=0; + for (uint i=lane*4u; i=total) return; uint m=gid/per, r=gid%per, d=r%hd, hh=hd/2; - uint base=m*per + (r/hd)*hd; // start of this (token,head) + uint base=m*per + (r/hd)*hd; float rot = (d0; s>>=1){ if (t0; s>>=1){ if (t0; s2>>=1){ if (t0; s2>>=1){ if (t0; s2>>=1){ if (t 0.0f ? 1.0f/gl : 0.0f; + for (uint d=t; d 0u) ? sinkN : 0u; uint R = ringR, Wsr = S + R; uint pos_first = q_pos0 + m0; @@ -1516,6 +1948,96 @@ kernel void kv_append_ring_i8_m(device const half* src [[buffer(0)]], device cha for (uint k=0;k 0.0f ? v/U.penalty : v*U.penalty); + } + } + threadgroup_barrier(mem_flags::mem_device); + for (uint i=t; i=n) return; + half v1 = (half)((float)in[i]/cap); + half v2 = (half)precise::tanh((float)v1); + y[i]=(half)((float)v2*cap); +} + +kernel void argmax_part(device const half* logits [[buffer(0)]], + device float* part [[buffer(1)]], + constant uint& V [[buffer(2)]], + constant uint& chunk [[buffer(3)]], + uint p [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* sb [[threadgroup(0)]], + threadgroup uint* si [[threadgroup(1)]], + threadgroup float* ss [[threadgroup(2)]]) { + uint lo = p*chunk, hi = min(V, lo+chunk); + float b = -INFINITY, s = -INFINITY; uint bi = lo; + for (uint i = lo+t; i < hi; i += T) { + float v = (float)logits[i]; + if (v > b) { s = b; b = v; bi = i; } + else if (v > s) { s = v; } + } + sb[t] = b; si[t] = bi; ss[t] = s; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = T >> 1; stride > 0u; stride >>= 1) { + if (t < stride) { + float ab = sb[t], asx = ss[t]; uint ai = si[t]; + float bb = sb[t+stride], bsx = ss[t+stride]; uint bidx = si[t+stride]; + if (ab > bb || (ab == bb && ai <= bidx)) { sb[t] = ab; si[t] = ai; ss[t] = max(asx, bb); } + else { sb[t] = bb; si[t] = bidx; ss[t] = max(bsx, ab); } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (t == 0u) { part[p*3u] = sb[0]; part[p*3u+1u] = ss[0]; part[p*3u+2u] = (float)si[0]; } +} + +kernel void argmax_final(device const float* part [[buffer(0)]], + device float* out3 [[buffer(1)]], + constant uint& NP [[buffer(2)]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* sb [[threadgroup(0)]], + threadgroup uint* si [[threadgroup(1)]], + threadgroup float* ss [[threadgroup(2)]]) { + float b = -INFINITY, s = -INFINITY; uint bi = 0u; + for (uint i = t; i < NP; i += T) { + float pb = part[i*3u], ps = part[i*3u+1u]; uint pi = (uint)part[i*3u+2u]; + if (pb > b || (pb == b && pi <= bi)) { s = max(s, b); b = pb; bi = pi; s = max(s, ps); } + else { s = max(s, pb); } + } + sb[t] = b; si[t] = bi; ss[t] = s; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = T >> 1; stride > 0u; stride >>= 1) { + if (t < stride) { + float ab = sb[t], asx = ss[t]; uint ai = si[t]; + float bb = sb[t+stride], bsx = ss[t+stride]; uint bidx = si[t+stride]; + if (ab > bb || (ab == bb && ai <= bidx)) { sb[t] = ab; si[t] = ai; ss[t] = max(asx, bb); } + else { sb[t] = bb; si[t] = bidx; ss[t] = max(bsx, ab); } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (t == 0u) { out3[0] = sb[0]; out3[1] = ss[0]; out3[2] = (float)si[0]; } +} + kernel void argmax_logits(device const half* logits [[buffer(0)]], device float* out3 [[buffer(1)]], constant uint& V [[buffer(2)]], diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 3d2000501..329c623e5 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -28,20 +28,22 @@ id psoT = nil; id psoTsimd = nil; id psoTbatch = nil; + id psoMega = nil, psoSwiT = nil, psoCat = nil; id psoG = nil, psoGmr = nil; id psoTm = nil, psoGm = nil; id psoGmma = nil, psoGdense = nil, psoDeq = nil; - id psoRotate = nil; + id psoRotate = nil, psoRotW = nil, psoEmbOW = nil, psoRmsAddRms = nil; id psoEmbO = nil, psoEmbH = nil; id psoEmbOm = nil, psoEmbHm = nil; id psoCopy=nil, psoBinary=nil, psoScalar=nil, psoUnary=nil, psoRms=nil, psoSwiglu=nil, psoRmsAdd=nil, psoRmsAddScale=nil; id psoCF16F32=nil, psoCF32F16=nil, psoCI8F16=nil, psoCF16I8=nil; - id psoAttn=nil, psoAttnC=nil, psoStrided=nil, psoScatter=nil, psoBcast=nil, psoKvAppend=nil; + id psoAttn=nil, psoAttnC=nil, psoAttnF=nil, psoStrided=nil, psoScatter=nil, psoBcast=nil, psoKvAppend=nil; id psoAttnPre=nil, psoAttnPreMma2=nil, psoAttnPreHd256=nil; id psoKvAppendM=nil, psoKvAppendRingM=nil; id psoSlideS=nil, psoSlideR=nil, psoSlideRM=nil; id psoRope=nil, psoRopeM=nil, psoRmsRope=nil; id psoArgmax=nil; + id psoArgmaxP=nil, psoArgmaxF=nil, psoSoftcap=nil, psoAdjust=nil; id psoGather=nil; id dummy=nil; bool ok = false; @@ -67,9 +69,10 @@ return p; }; psoT=pso("cq4_transform"); psoTsimd=pso("cq4_transform_simd"); psoG=pso("cq4_gemv"); psoTbatch=pso("cq4_transform_batch"); psoGmr=pso("cq4_gemv_mr"); + psoMega=pso("cq4_gemv_mega"); psoSwiT=pso("cq4_swiglu_transform"); psoCat=pso("cq4_gemv_cat"); psoTm=pso("cq4_transform_m"); psoGm=pso("cq4_gemm"); psoGmma=pso("cq4_gemm_mma"); psoGdense=pso("cq4_gemm_dense_f16"); psoDeq=pso("cq4_dequant_f16"); - psoRotate=pso("lmhead_rotate"); + psoRotate=pso("lmhead_rotate"); psoRotW=pso("lmhead_rotate_wide"); psoEmbOW=pso("emb_ortho_wide"); psoRmsAddRms=pso("rms_norm_add_rms_f16"); psoEmbO=pso("emb_ortho"); psoEmbH=pso("emb_hadamard"); psoEmbOm=pso("emb_ortho_m"); psoEmbHm=pso("emb_hadamard_m"); psoGather=pso("gather_f16"); @@ -77,13 +80,15 @@ psoUnary=pso("unary_f16"); psoRms=pso("rms_norm_f16"); psoSwiglu=pso("swiglu_f16"); psoRmsAdd=pso("rms_norm_add_f16"); psoRmsAddScale=pso("rms_norm_add_scale_f16"); psoCF16F32=pso("cast_f16_f32"); psoCF32F16=pso("cast_f32_f16"); psoCI8F16=pso("cast_i8_f16"); psoCF16I8=pso("cast_f16_i8"); - psoAttn=pso("attn_decode_i8"); psoAttnC=pso("attn_decode_combine"); + psoAttn=pso("attn_decode_i8"); psoAttnC=pso("attn_decode_combine"); psoAttnF=pso("attn_decode_fused_i8"); psoStrided=pso("strided_copy_f16"); psoBcast=pso("bcast_binary_f16"); psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoAttnPreHd256=pso("attn_prefill_mma_hd256"); psoKvAppendM=pso("kv_append_i8_m"); psoKvAppendRingM=pso("kv_append_ring_i8_m"); psoSlideS=pso("kv_slide_save"); psoSlideR=pso("kv_slide_restore"); psoSlideRM=pso("kv_slide_restore_m"); psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); psoRopeM=pso("rope_f16_m"); psoRmsRope=pso("rms_norm_rope_f16"); psoArgmax=pso("argmax_logits"); + psoArgmaxP=pso("argmax_part"); psoArgmaxF=pso("argmax_final"); psoSoftcap=pso("softcap_f16"); + psoAdjust=pso("adjust_logits"); if (prof) fprintf(stderr,"[prof-init] pso creation: %.1fms total\n", ms()); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 @@ -101,9 +106,10 @@ struct ResW { id packed=nil, norms=nil, codebook=nil, lsign=nil, rsign=nil, perm=nil, recip=nil; id rotation=nil; - id wf16=nil; // lazily-dequantized dense fp16 weight (prefill pre-dequant path) + id wf16=nil; }; std::unordered_map g_resident; +std::unordered_map g_resident_emb; std::mutex g_resident_mu; void deinterleave_4row(const CactusQuantMatrix* W, @@ -165,21 +171,20 @@ void deinterleave_4row(const CactusQuantMatrix* W, inline size_t bucket(size_t b) { return (b + 4095) & ~size_t(4095); } struct GemmProf { - double gpu_ns=0; long cmdbufs=0; // non-perturbing: total GPU-busy across all committed command buffers + double gpu_ns=0; long cmdbufs=0; ~GemmProf(){ if(std::getenv("CACTUS_PROF_GPU") && cmdbufs>0) fprintf(stderr,"[PROF gpu-busy] cmdbufs=%ld total_gpu=%.2fms\n", cmdbufs, gpu_ns/1e6); } }; static GemmProf g_gemmprof; static const bool g_prof_gpu = (std::getenv("CACTUS_PROF_GPU") != nullptr); -// Per-op-type GPU-time profiler: flush+time each dispatch, attribute to the current op tag g_op. -// Perturbing (serializes), but gives an exact per-category GPU-time breakdown of a forward pass. static const bool g_prof_ops = (std::getenv("CACTUS_PROF_OPS") != nullptr); -static const bool g_gemv_mr = (std::getenv("CACTUS_NO_GEMV_MR") == nullptr); // multi-row M=1 gemv (default on; CACTUS_NO_GEMV_MR disables) -static const uint32_t g_mr_rows = 8u * 2u; // ROWS * CQ4_NR — MUST match cactus_kernels.metal CQ4_NR +static const bool g_gemv_mr = (std::getenv("CACTUS_NO_GEMV_MR") == nullptr); +static const uint32_t g_mr_rows = 8u * 2u; + static const bool g_skip_mm=(std::getenv("CACTUS_SKIP_MM")!=nullptr), g_skip_attn=(std::getenv("CACTUS_SKIP_ATTN")!=nullptr), g_skip_norm=(std::getenv("CACTUS_SKIP_NORM")!=nullptr); static const bool g_skip_transform=(std::getenv("CACTUS_SKIP_TRANSFORM")!=nullptr); -static const bool g_predequant=(std::getenv("CACTUS_NO_PREDEQUANT")==nullptr); // prefill: dense fp16 gemm on pre-dequantized weights (default on; +~1.25GB RAM) +static const bool g_predequant=(std::getenv("CACTUS_NO_PREDEQUANT")==nullptr); static const char* g_op = "other"; struct OpProf { std::map> t; @@ -192,19 +197,28 @@ void deinterleave_4row(const CactusQuantMatrix* W, static OpProf g_opprof; static const bool g_concurrent = (std::getenv("CACTUS_GPU_CONCURRENT") != nullptr); +static const bool g_no_groups = (std::getenv("CACTUS_NO_GROUPS") != nullptr); +bool g_session_concurrent = false; +bool g_group = false; id ensureEncoder() { if (!g_enc) { g_cmd = [ctx().queue commandBuffer]; - g_enc = g_concurrent ? [g_cmd computeCommandEncoderWithDispatchType:MTLDispatchTypeConcurrent] + g_enc = (g_concurrent || g_session_concurrent) + ? [g_cmd computeCommandEncoderWithDispatchType:MTLDispatchTypeConcurrent] : [g_cmd computeCommandEncoder]; } return g_enc; } +inline void hard_barrier() { + if ((g_concurrent || g_session_concurrent) && g_enc) + [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; +} + inline void barrier() { static const bool force = (std::getenv("CACTUS_FORCE_BARRIER") != nullptr); - if (g_prof_ops && g_enc) { // flush + time this single dispatch, attribute to g_op + if (g_prof_ops && g_enc) { [g_enc endEncoding]; [g_cmd commit]; [g_cmd waitUntilCompleted]; g_opprof.t[g_op].first += (g_cmd.GPUEndTime - g_cmd.GPUStartTime)*1e9; g_opprof.t[g_op].second += 1; @@ -214,7 +228,8 @@ inline void barrier() { g_op = "other"; return; } - if (g_concurrent || force) [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; + if (g_group) return; + if (g_concurrent || g_session_concurrent || force) [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; } id recycled(size_t bytes) { @@ -227,6 +242,25 @@ inline void barrier() { return b; } +std::map> g_wrapped; +std::unordered_map> g_readonly; +std::pair, size_t> wrapHostPtr(const void* p, size_t bytes) { + uintptr_t a = (uintptr_t)p, base = a & ~(uintptr_t)16383u; + size_t need = (a - base) + (bytes ? bytes : 1); + auto it = g_wrapped.upper_bound(a); + if (it != g_wrapped.begin()) { + --it; + if (a >= it->first && a + (bytes?bytes:1) <= it->first + (size_t)it->second.length) + return { it->second, (size_t)(a - it->first) }; + } + size_t wraplen = (need + 16383u) & ~(size_t)16383u; + id b = [ctx().dev newBufferWithBytesNoCopy:(void*)base length:wraplen + options:MTLResourceStorageModeShared deallocator:nil]; + if (!b) return { nil, 0 }; + g_wrapped[base] = b; + return { b, (size_t)(a - base) }; +} + std::pair, size_t> bufForPtrOff(const void* p, size_t bytes) { uintptr_t a = reinterpret_cast(p); auto it = g_shared.upper_bound(a); @@ -237,6 +271,9 @@ inline void barrier() { return { it->second, static_cast(a - base) }; } + auto wr = wrapHostPtr(p, bytes); + if (wr.first) return wr; + size_t nb = bytes ? bytes : 1; id b = recycled(bytes); std::memcpy([b contents], p, nb); @@ -256,7 +293,16 @@ inline void setBufAt(const void* p, size_t bytes, int idx) { bool cactus_metal_active_mode() { return ctx().ok && g_active; } bool cactus_metal_concurrent() { return g_concurrent; } -void cactus_metal_barrier() { if (g_concurrent && g_enc) [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; } +void cactus_metal_barrier() { if ((g_concurrent || g_session_concurrent) && g_enc) [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; } + +void cactus_metal_session_set_concurrent(bool on) { + if (g_no_groups) return; + if (g_enc) return; + g_session_concurrent = on; + if (!on) g_group = false; +} +void cactus_metal_group_begin() { if (g_session_concurrent && !g_prof_ops) g_group = true; } +void cactus_metal_group_end() { if (g_group) { g_group = false; barrier(); } } void cactus_metal_session_begin() {} void cactus_metal_session_sync() { @@ -273,6 +319,23 @@ void cactus_metal_session_sync() { void cactus_metal_session_end() { cactus_metal_session_sync(); } +void cactus_metal_invalidate_host_wraps() { + std::lock_guard lk(g_resident_mu); + g_wrapped.clear(); + g_readonly.clear(); + g_resident.clear(); + g_resident_emb.clear(); +} + +void cactus_metal_session_flush() { + if (!g_enc) return; + [g_enc endEncoding]; + if (g_prof_gpu) [g_cmd addCompletedHandler:^(id cb){ + g_gemmprof.gpu_ns += (cb.GPUEndTime-cb.GPUStartTime)*1e9; g_gemmprof.cmdbufs += 1; }]; + [g_cmd commit]; + g_enc = nil; g_cmd = nil; +} + void* cactus_metal_alloc_shared(size_t bytes) { if (!ctx().ok) return nullptr; @@ -434,11 +497,53 @@ bool cactus_metal_encode_rms_norm_add_scale(void* out, const void* in, const voi barrier(); return true; } +bool cactus_metal_encode_rms_norm_add_rms(void* h_out, void* xn_out, const void* in, const void* w1, + const void* res, const void* w2, + size_t rows, size_t dim, float eps, float out_scale) { + if (!ctx().ok || !ctx().psoRmsAddRms) return false; + ensureEncoder(); + g_op="rmsnorm"; + uint32_t d=(uint32_t)dim; float e=eps, os=out_scale; + [g_enc setComputePipelineState:ctx().psoRmsAddRms]; + setBufAt(in, rows*dim*2, 0); setBufAt(w1, dim*2, 1); setBufAt(res, rows*dim*2, 2); + setBufAt(h_out, rows*dim*2, 3); setBufAt(w2, dim*2, 4); setBufAt(xn_out, rows*dim*2, 5); + [g_enc setBytes:&d length:4 atIndex:6]; [g_enc setBytes:&e length:4 atIndex:7]; [g_enc setBytes:&os length:4 atIndex:8]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + if(!g_skip_norm) [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} + bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) { if (!ctx().ok) return false; ensureEncoder(); g_op="argmax"; - uint32_t V = vocab, T = 1024; + uint32_t V = vocab; + if (ctx().psoArgmaxP && ctx().psoArgmaxF && V >= 32768u) { + const uint32_t NP = 128u, T = 256u; + uint32_t chunk = (V + NP - 1u)/NP; + id part = recycled((size_t)NP*3*sizeof(float)); + [g_enc setComputePipelineState:ctx().psoArgmaxP]; + setBufAt(logits, (size_t)vocab*2, 0); + [g_enc setBuffer:part offset:0 atIndex:1]; + [g_enc setBytes:&V length:4 atIndex:2]; [g_enc setBytes:&chunk length:4 atIndex:3]; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:T*sizeof(uint) atIndex:1]; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:2]; + [g_enc dispatchThreadgroups:MTLSizeMake(NP,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + hard_barrier(); + [g_enc setComputePipelineState:ctx().psoArgmaxF]; + [g_enc setBuffer:part offset:0 atIndex:0]; + setBufAt(out3, 3*sizeof(float), 1); + [g_enc setBytes:&NP length:4 atIndex:2]; + [g_enc setThreadgroupMemoryLength:NP*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:NP*sizeof(uint) atIndex:1]; + [g_enc setThreadgroupMemoryLength:NP*sizeof(float) atIndex:2]; + [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(NP,1,1)]; + barrier(); + return true; + } + uint32_t T = 1024; [g_enc setComputePipelineState:ctx().psoArgmax]; setBufAt(logits, (size_t)vocab*2, 0); setBufAt(out3, 3*sizeof(float), 1); @@ -450,6 +555,42 @@ bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) barrier(); return true; } +bool cactus_metal_encode_adjust_logits(void* logits, size_t vocab, + const uint32_t* recent, uint32_t n_recent, + const uint32_t* bias_ids, const float* bias_vals, uint32_t n_bias, + int64_t suppressed, float penalty) { + if (!ctx().ok || !ctx().psoAdjust) return false; + if (n_recent > 4096u || n_bias > 65536u) return false; + ensureEncoder(); + g_op="other"; + id rb = ctx().dummy, bi = ctx().dummy, bv = ctx().dummy; + if (n_recent) { rb = recycled((size_t)n_recent*4); std::memcpy([rb contents], recent, (size_t)n_recent*4); } + if (n_bias) { bi = recycled((size_t)n_bias*4); std::memcpy([bi contents], bias_ids, (size_t)n_bias*4); + bv = recycled((size_t)n_bias*4); std::memcpy([bv contents], bias_vals, (size_t)n_bias*4); } + struct { uint32_t n_recent, n_bias, suppress_flag, suppress_id; float penalty; } U = + { n_recent, n_bias, + (uint32_t)(suppressed >= 0 ? 1 : 0), (uint32_t)(suppressed >= 0 ? suppressed : 0), penalty }; + [g_enc setComputePipelineState:ctx().psoAdjust]; + setBufAt(logits, vocab*2, 0); + [g_enc setBuffer:rb offset:0 atIndex:1]; + [g_enc setBuffer:bi offset:0 atIndex:2]; [g_enc setBuffer:bv offset:0 atIndex:3]; + [g_enc setBytes:&U length:sizeof(U) atIndex:4]; + [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(1024,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_softcap(void* out, const void* in, size_t n, float cap) { + if (!ctx().ok || !ctx().psoSoftcap) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; float c=cap; + [g_enc setComputePipelineState:ctx().psoSoftcap]; + setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&c length:4 atIndex:3]; + [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_prec, size_t n) { if (!ctx().ok) return false; @@ -499,7 +640,7 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu [g_enc setBuffer:rw.perm offset:0 atIndex:4]; [g_enc setBuffer:g_code_buf offset:0 atIndex:5]; [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; - barrier(); + hard_barrier(); bool umr = g_gemv_mr && ctx().psoGmr && (N % g_mr_rows == 0u); [g_enc setComputePipelineState:(umr?ctx().psoGmr:ctx().psoG)]; [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:0 atIndex:1]; @@ -515,7 +656,6 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu return true; } -// Batched CQ4 transform for up to 3 weights sharing the input `x` (recip). One dispatch instead of B. bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* const* Ws, int B, void* const* codes) { if (!ctx().ok || B < 1 || B > 3 || !ctx().psoTbatch) return false; const uint32_t gs=Ws[0]->group_size, K=Ws[0]->K, ng=Ws[0]->num_groups; @@ -542,7 +682,6 @@ bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* return true; } -// CQ4 gemv with a pre-computed transformed activation `code` (no transform dispatch here). bool cactus_metal_encode_gemv_precoded(void* out, const void* code, const CactusQuantMatrix* W) { if (!ctx().ok) return false; const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; @@ -564,6 +703,106 @@ bool cactus_metal_encode_gemv_precoded(void* out, const void* code, const Cactus return true; } +bool cactus_metal_encode_gemv_mega_ex(void* const* outs, const void* x, const void* normw, float eps, + const CactusQuantMatrix* const* Ws, int B, const void* osw, + bool pswi, float pscale) { + if (!ctx().ok || !ctx().psoMega || B < 1 || B > 3) return false; + const uint32_t gs=Ws[0]->group_size, K=Ws[0]->K, ng=Ws[0]->num_groups; + if (gs != 128u) return false; + uint32_t Ns[3]={0,0,0}; + for (int i=0;iK != K || (Ws[i]->N % 16u) != 0u) return false; + Ns[i]=Ws[i]->N; + } + ResW* rw[3]; rw[0]=&resident(Ws[0]); rw[1]=rw[0]; rw[2]=rw[0]; + for (int i=1;irecip offset:0 atIndex:2]; + for (int i=0;i<3;i++){ + ResW* R=rw[i]; + [g_enc setBuffer:R->lsign offset:0 atIndex:3+i]; + [g_enc setBuffer:R->rsign offset:0 atIndex:6+i]; + [g_enc setBuffer:R->perm offset:0 atIndex:9+i]; + [g_enc setBuffer:R->codebook offset:0 atIndex:12+i]; + [g_enc setBuffer:R->norms offset:0 atIndex:15+i]; + [g_enc setBuffer:R->packed offset:0 atIndex:18+i]; + setBufAt(outs[igroup_size != 128u || !quant_fast_eligible(W)) return false; + const uint32_t K=W->K, ng=W->num_groups; + ResW& rw = resident(W); + ensureEncoder(); + g_op="transform"; + float s = scale; + [g_enc setComputePipelineState:ctx().psoSwiT]; + setBufAt(gate, (size_t)K*2, 0); setBufAt(up, (size_t)K*2, 1); + [g_enc setBuffer:rw.recip offset:0 atIndex:2]; + [g_enc setBuffer:rw.lsign offset:0 atIndex:3]; [g_enc setBuffer:rw.rsign offset:0 atIndex:4]; + [g_enc setBuffer:rw.perm offset:0 atIndex:5]; + setBufAt(code, (size_t)K*2, 6); + [g_enc setBytes:&s length:4 atIndex:7]; + [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + barrier(); + return true; +} + +bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, + const CactusQuantMatrix* const* Ws, int B) { + if (!ctx().ok || !ctx().psoCat || B < 1 || B > 3) return false; + const uint32_t gs=Ws[0]->group_size, K=Ws[0]->K, ng=Ws[0]->num_groups; + if (gs != 128u) return false; + uint32_t Ns[3]={0,0,0}; + for (int i=0;iK != K || (Ws[i]->N % 16u) != 0u) return false; + Ns[i]=Ws[i]->N; + } + ResW* rw[3]; rw[0]=&resident(Ws[0]); rw[1]=rw[0]; rw[2]=rw[0]; + for (int i=1;ipacked offset:0 atIndex:3+i]; + [g_enc setBuffer:rw[j]->codebook offset:0 atIndex:6+i]; + [g_enc setBuffer:rw[j]->norms offset:0 atIndex:9+i]; + setBufAt(outs[j], (size_t)Ns[j]*2, 12+i); + } + [g_enc setBytes:&U length:sizeof(U) atIndex:15]; + uint32_t total = Ns[0]+Ns[1]+Ns[2]; + if(!g_skip_mm)[g_enc dispatchThreadgroups:MTLSizeMake(total/16u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; +} + bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { if (!ctx().ok) return false; if (!quant_fast_eligible(W)) return false; @@ -591,11 +830,8 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; g_op="transform"; if (!g_skip_transform) [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; - barrier(); + hard_barrier(); - // Pre-dequant dense path: dequantize the weight to fp16 ONCE (cached in rw.wf16), then run a plain - // fp16 SG_MAT gemm — avoids the codebook LUT in the gemm fill (~33% faster gemm). The one-time dequant - // is triggered inside the load-time prefill warmup, so real prefill only pays the fast dense gemm. if (g_predequant && ctx().psoGdense && ctx().psoDeq) { if (!rw.wf16) { rw.wf16 = [ctx().dev newBufferWithLength:(size_t)N*K*2 options:MTLResourceStorageModeShared]; @@ -605,7 +841,7 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus [g_enc setBytes:&gs length:4 atIndex:4]; [g_enc setBytes:&ng length:4 atIndex:5]; [g_enc setBytes:&pgb length:4 atIndex:6]; [g_enc setBytes:&N length:4 atIndex:7]; [g_enc dispatchThreads:MTLSizeMake((size_t)N*ng,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); + hard_barrier(); } [g_enc setComputePipelineState:ctx().psoGdense]; [g_enc setBuffer:g_code_buf_m offset:0 atIndex:0]; [g_enc setBuffer:rw.wf16 offset:0 atIndex:1]; @@ -644,12 +880,21 @@ bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* co if (!rw.rotation || !rw.packed) return false; ensureEncoder(); - [g_enc setComputePipelineState:ctx().psoRotate]; - setBufAt(act, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; - [g_enc setBuffer:rw.rotation offset:0 atIndex:2]; setBufAt(code, (size_t)K*2, 3); - [g_enc setBytes:&K length:4 atIndex:4]; - [g_enc dispatchThreads:MTLSizeMake(K,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); + if (ctx().psoRotW) { + [g_enc setComputePipelineState:ctx().psoRotW]; + setBufAt(act, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; + [g_enc setBuffer:rw.rotation offset:0 atIndex:2]; setBufAt(code, (size_t)K*2, 3); + [g_enc setBytes:&K length:4 atIndex:4]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((K+31)/32,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } else { + [g_enc setComputePipelineState:ctx().psoRotate]; + setBufAt(act, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; + [g_enc setBuffer:rw.rotation offset:0 atIndex:2]; setBufAt(code, (size_t)K*2, 3); + [g_enc setBytes:&K length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(K,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } + hard_barrier(); bool umr = g_gemv_mr && ctx().psoGmr && (N % g_mr_rows == 0u); [g_enc setComputePipelineState:(umr?ctx().psoGmr:ctx().psoG)]; @@ -665,13 +910,26 @@ bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* co return true; } -bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W) { +bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W, float scale) { if (!ctx().ok || !W->rotation || W->bits != 4) return false; const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size; if (ng != 1 || gs != K) return false; ResW& rw = resident(W); if (!rw.packed || !rw.rotation || !rw.codebook || !rw.norms || !rw.recip) return false; ensureEncoder(); + if (!(ctx().psoEmbOW && (K % 8u) == 0u) && scale != 1.0f) return false; + if (ctx().psoEmbOW && (K % 8u) == 0u) { + [g_enc setComputePipelineState:ctx().psoEmbOW]; + [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; + [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.recip offset:0 atIndex:3]; + [g_enc setBuffer:rw.rotation offset:0 atIndex:4]; setBufAt(out, (size_t)K*2, 5); + [g_enc setBytes:&K length:4 atIndex:6]; [g_enc setBytes:&row length:4 atIndex:7]; + [g_enc setBytes:&scale length:4 atIndex:8]; + [g_enc setThreadgroupMemoryLength:(size_t)K*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(K/8u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; + } [g_enc setComputePipelineState:ctx().psoEmbO]; [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.recip offset:0 atIndex:3]; @@ -684,7 +942,6 @@ bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQu return true; } -std::unordered_map g_resident_emb; ResW& resident_emb_meta(const CactusQuantMatrix* W) { std::lock_guard lk(g_resident_mu); auto it = g_resident_emb.find(W->packed_indices); @@ -724,6 +981,18 @@ bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const Cactu return true; } +static id registerReadonly(const void* p, size_t bytes) { + if (!p) return nil; + auto it = g_readonly.find(p); + if (it != g_readonly.end()) return it->second; + uintptr_t a = (uintptr_t)p, base = a & ~(uintptr_t)16383u; + size_t wraplen = (((a - base) + bytes + 16383u) & ~(size_t)16383u); + id b = [ctx().dev newBufferWithBytesNoCopy:(void*)base length:wraplen + options:MTLResourceStorageModeShared deallocator:nil]; + if (b) g_readonly[p] = b; + return b; +} + bool cactus_metal_encode_embedding_ortho_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M) { if (!ctx().ok || !ctx().psoEmbOm || !W->rotation || W->bits != 4) return false; const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size; @@ -770,18 +1039,6 @@ bool cactus_metal_encode_embedding_hadamard_m(void* out, const CactusQuantMatrix return true; } -static std::unordered_map> g_readonly; -static id registerReadonly(const void* p, size_t bytes) { - if (!p) return nil; - auto it = g_readonly.find(p); - if (it != g_readonly.end()) return it->second; - uintptr_t a = (uintptr_t)p, base = a & ~(uintptr_t)16383u; - size_t wraplen = (((a - base) + bytes + 16383u) & ~(size_t)16383u); - id b = [ctx().dev newBufferWithBytesNoCopy:(void*)base length:wraplen - options:MTLResourceStorageModeShared deallocator:nil]; - if (b) g_readonly[p] = b; - return b; -} bool cactus_metal_encode_gather_f16(void* out, const void* table, size_t table_bytes, const uint32_t* rows, uint32_t M, uint32_t D) { @@ -831,7 +1088,7 @@ bool cactus_metal_encode_attention_i8( const uint32_t T = 256, nsg = T / 32u; const uint32_t R = kv_end - kv_start; - uint32_t nwg = (R <= 256u) ? 1u : R / 96u; if (nwg > 32u) nwg = 32u; + uint32_t nwg = R / 24u; if (nwg < 1u) nwg = 1u; if (nwg > 32u) nwg = 32u; id partO = ctx().dummy, partML = ctx().dummy; if (nwg > 1u) { partO = recycled((size_t)num_q_heads*nwg*v_hdim*sizeof(float)); @@ -848,7 +1105,7 @@ bool cactus_metal_encode_attention_i8( [g_enc setBytes:&nwg length:4 atIndex:18]; [g_enc setThreadgroupMemoryLength:((size_t)nsg*v_hdim + 2*nsg)*sizeof(float) atIndex:0]; if(!g_skip_attn)[g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; - barrier(); + hard_barrier(); if (nwg > 1u) { [g_enc setComputePipelineState:ctx().psoAttnC]; [g_enc setBuffer:partO offset:0 atIndex:0]; [g_enc setBuffer:partML offset:0 atIndex:1]; @@ -860,6 +1117,56 @@ bool cactus_metal_encode_attention_i8( return true; } +bool cactus_metal_encode_attention_fused_i8( + void* out, const void* q, const void* kraw, const void* vraw, + void* kc, void* vc, void* ks, void* vs, + const void* qw, const void* kw, const void* vw, const void* cs, const void* sn, + uint32_t nqh, uint32_t hd, uint32_t vhd, + uint32_t kv_start, uint32_t kv_end, uint32_t slot, uint32_t has_new, + float eps, float scale, + size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes) { + if (!ctx().ok || !ctx().psoAttnF) return false; + if (kv_end <= kv_start) return false; + ensureEncoder(); + g_op="attention"; + const uint32_t T = 256, nsg = T / 32u; + const uint32_t R = kv_end - kv_start; + uint32_t nwg = R / 24u; if (nwg < 1u) nwg = 1u; if (nwg > 32u) nwg = 32u; + id partO = ctx().dummy, partML = ctx().dummy; + if (nwg > 1u) { + partO = recycled((size_t)nqh*nwg*vhd*sizeof(float)); + partML = recycled((size_t)nqh*nwg*2*sizeof(float)); + } + struct { uint32_t nqh, nkvh, hd, vhd, hist, kv_start, kv_end, nwg, slot, has_new; float eps, scale; } U = + { nqh, 1u, hd, vhd, 0u, kv_start, kv_end, nwg, slot, has_new, eps, scale }; + [g_enc setComputePipelineState:ctx().psoAttnF]; + setBufAt(q, (size_t)nqh*hd*2, 0); + if (has_new && kraw && vraw) { setBufAt(kraw, (size_t)hd*2, 1); setBufAt(vraw, (size_t)vhd*2, 2); } + else { [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; } + setBufAt(kc, kc_bytes, 3); setBufAt(vc, vc_bytes, 4); + setBufAt(ks, ks_bytes, 5); setBufAt(vs, vs_bytes, 6); + setBufAt(out, (size_t)nqh*vhd*2, 7); + setBufAt(qw, (size_t)hd*2, 8); + if (has_new && kw) setBufAt(kw, (size_t)hd*2, 9); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:9]; + if (has_new && vw) setBufAt(vw, (size_t)vhd*2, 10); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:10]; + setBufAt(cs, (size_t)hd*2, 11); setBufAt(sn, (size_t)hd*2, 12); + [g_enc setBuffer:partO offset:0 atIndex:13]; [g_enc setBuffer:partML offset:0 atIndex:14]; + [g_enc setBytes:&U length:sizeof(U) atIndex:15]; + [g_enc setThreadgroupMemoryLength:((size_t)nsg*vhd + 2*nsg + 256)*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:(size_t)(2*hd + vhd)*2 atIndex:1]; + if(!g_skip_attn)[g_enc dispatchThreadgroups:MTLSizeMake(nqh*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + hard_barrier(); + if (nwg > 1u) { + [g_enc setComputePipelineState:ctx().psoAttnC]; + [g_enc setBuffer:partO offset:0 atIndex:0]; [g_enc setBuffer:partML offset:0 atIndex:1]; + setBufAt(out, (size_t)nqh*vhd*2, 2); + [g_enc setBytes:&vhd length:4 atIndex:3]; [g_enc setBytes:&nwg length:4 atIndex:4]; + [g_enc dispatchThreadgroups:MTLSizeMake(nqh,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } + barrier(); + return true; +} + bool cactus_metal_encode_attention_i8_prefill( void* out, const void* q, const void* knew, const void* vnew, const void* kc, const void* vc, const void* ks, const void* vs, @@ -872,15 +1179,10 @@ bool cactus_metal_encode_attention_i8_prefill( const uint32_t T = 128; if (total_keys == 0 || M == 0 || (num_q_heads % num_kv_heads) != 0) return false; uint32_t maxsc = (ring > 0u) ? ((total_keys > sink + ring) ? (sink + ring) : total_keys) : 256u; - static const bool en_hd256 = (std::getenv("CACTUS_NO_HD256") == nullptr); // tiled local (hd=256) prefill attention (default on) + static const bool en_hd256 = (std::getenv("CACTUS_NO_HD256") == nullptr); bool mma2 = (ring == 0u && is_causal && total_keys >= 2048u && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u); - // Route ALL global (hd=512, full-attention / window==0) layers through the tiled mma2 kernel, not just - // the long-context ones — the tiled kernel loads each KV tile into shared once and reuses it across a - // block of 8 queries, vs the scalar path re-reading the whole KV cache per query. Always correct for - // window==0 (no sliding-window eviction). Biggest prefill attention win (pp2048 +22%). if (!mma2 && is_causal && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u && window == 0u) mma2 = true; - // hd256 now handles the sliding-window (sink + ring slot mapping), so it's valid for local layers too. bool hd256 = (en_hd256 && ctx().psoAttnPreHd256 != nil && is_causal && head_dim == 256u && v_hdim == 256u && num_q_heads == 8u && num_kv_heads == 1u); if (mma2 || hd256) { @@ -1044,7 +1346,7 @@ bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, v [g_enc setBytes:&group_size length:4 atIndex:6]; [g_enc setBytes:&shift_src length:4 atIndex:7]; [g_enc setBytes:&remaining length:4 atIndex:8]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; - barrier(); + hard_barrier(); } uint32_t n2 = (remaining+1)*per; [g_enc setComputePipelineState:ctx().psoSlideR]; @@ -1076,7 +1378,7 @@ bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, [g_enc setBytes:&group_size length:4 atIndex:6]; [g_enc setBytes:&shift_src length:4 atIndex:7]; [g_enc setBytes:&remaining length:4 atIndex:8]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; - barrier(); + hard_barrier(); } uint32_t n2 = (remaining+M)*per; [g_enc setComputePipelineState:ctx().psoSlideRM]; diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 24132cf21..b351a91b6 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -7,12 +7,29 @@ void cactus_metal_set_active(bool) {} bool cactus_metal_active_mode() { return false; } bool cactus_metal_concurrent() { return false; } void cactus_metal_barrier() {} +void cactus_metal_session_set_concurrent(bool) {} +void cactus_metal_group_begin() {} +void cactus_metal_group_end() {} +bool cactus_metal_encode_softcap(void*, const void*, size_t, float) { return false; } +bool cactus_metal_encode_adjust_logits(void*, size_t, const uint32_t*, uint32_t, + const uint32_t*, const float*, uint32_t, + int64_t, float) { return false; } +bool cactus_metal_encode_gemv_mega(void* const*, const void*, const void*, float, + const CactusQuantMatrix* const*, int, const void*) { return false; } +bool cactus_metal_encode_gemv_mega_ex(void* const*, const void*, const void*, float, + const CactusQuantMatrix* const*, int, const void*, bool, float) { return false; } +bool cactus_metal_encode_gemv_cat(void* const*, const void* const*, + const CactusQuantMatrix* const*, int) { return false; } +bool cactus_metal_encode_swiglu_transform(void*, const void*, const void*, + const CactusQuantMatrix*, float) { return false; } void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, uint32_t M, __fp16* C) { cactus_quant_matmul(W, A, M, C); } void cactus_metal_session_begin() {} void cactus_metal_session_sync() {} +void cactus_metal_session_flush() {} +void cactus_metal_invalidate_host_wraps() {} void cactus_metal_session_end() {} void* cactus_metal_alloc_shared(size_t) { return nullptr; } void cactus_metal_free_shared(void*) {} @@ -26,6 +43,8 @@ bool cactus_metal_encode_rope_m(void*, const void*, const void*, const void*, ui bool cactus_metal_encode_rms_norm(void*, const void*, const void*, size_t, size_t, float) { return false; } bool cactus_metal_encode_rms_norm_rope(void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, float) { return false; } bool cactus_metal_encode_rms_norm_add(void*, const void*, const void*, const void*, size_t, size_t, float) { return false; } +bool cactus_metal_encode_rms_norm_add_rms(void*, void*, const void*, const void*, const void*, const void*, + size_t, size_t, float, float) { return false; } bool cactus_metal_encode_rms_norm_add_scale(void*, const void*, const void*, const void*, size_t, size_t, float, float) { return false; } bool cactus_metal_encode_argmax(const void*, uint32_t, void*) { return false; } bool cactus_metal_encode_cast(void*, int, const void*, int, size_t) { return false; } @@ -35,7 +54,7 @@ bool cactus_metal_encode_transform_batch(const void*, const CactusQuantMatrix* c bool cactus_metal_encode_gemv_precoded(void*, const void*, const CactusQuantMatrix*) { return false; } bool cactus_metal_prewarm_quant(const CactusQuantMatrix*) { return false; } bool cactus_metal_encode_quant_matmul_ortho(void*, const void*, void*, const CactusQuantMatrix*) { return false; } -bool cactus_metal_encode_embedding_ortho(void*, uint32_t, const CactusQuantMatrix*) { return false; } +bool cactus_metal_encode_embedding_ortho(void*, uint32_t, const CactusQuantMatrix*, float) { return false; } bool cactus_metal_encode_embedding_hadamard(void*, uint32_t, const CactusQuantMatrix*) { return false; } bool cactus_metal_encode_embedding_ortho_m(void*, const CactusQuantMatrix*, const uint32_t*, uint32_t) { return false; } bool cactus_metal_encode_embedding_hadamard_m(void*, const CactusQuantMatrix*, const uint32_t*, uint32_t) { return false; } @@ -43,6 +62,10 @@ bool cactus_metal_encode_gather_f16(void*, const void*, size_t, const uint32_t*, bool cactus_metal_encode_attention_i8(void*, const void*, const void*, const void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, size_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_attention_fused_i8(void*, const void*, const void*, const void*, + void*, void*, void*, void*, const void*, const void*, const void*, const void*, const void*, + uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, + float, float, size_t, size_t, size_t, size_t) { return false; } bool cactus_metal_encode_attention_i8_prefill(void*, const void*, const void*, const void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, size_t, size_t, size_t, size_t, uint32_t, uint32_t) { return false; } From 528ebe5664b3a024bb96fe0226f8fbdcc85254dd Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 05:46:46 +0200 Subject: [PATCH 10/36] cq2/3 Signed-off-by: jakmro --- cactus-kernels/src/cactus_kernels.metal | 174 ++++++++++++++++++++++-- cactus-kernels/src/metal_backend.mm | 158 +++++++++++++++++---- 2 files changed, 293 insertions(+), 39 deletions(-) diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 4953770d6..35b6f93e7 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -2,6 +2,16 @@ using namespace metal; #define ROWS 8u +inline uint cq_idx(device const uchar* packed, uint k, uint bits) { + if (bits == 4u) return (packed[k >> 1] >> ((k & 1u) * 4u)) & 0xFu; + if (bits == 2u) return (packed[k >> 2] >> ((k & 3u) * 2u)) & 0x3u; + uint bit_pos = k * 3u; + uint byte_idx = bit_pos >> 3, bit_idx = bit_pos & 7u; + uint val = (uint)packed[byte_idx] >> bit_idx; + if (bit_idx > 5u) val |= (uint)packed[byte_idx + 1u] << (8u - bit_idx); + return val & 0x7u; +} + inline float gelu_tanh(float x) { float c = 0.7978845608028654f * (x + 0.044715f*x*x*x); return 0.5f * x * (1.0f + precise::tanh(c)); @@ -421,6 +431,149 @@ kernel void cq4_gemv_cat( } } +kernel void cq_gemv_mr_lowbit( + device const half* code [[buffer(0)]], + device const uchar* packed [[buffer(1)]], + device const half* codebook [[buffer(2)]], + device const half* norms [[buffer(3)]], + device half* y [[buffer(4)]], + constant uint& gs [[buffer(5)]], + constant uint& num_groups [[buffer(6)]], + constant uint& pgb [[buffer(7)]], + constant uint& N [[buffer(8)]], + constant uint& bits [[buffer(9)]], + uint tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]]) +{ + threadgroup float cb[8]; + uint cbn = 1u << bits; + if (tl < cbn) cb[tl] = (float)codebook[tl]; + threadgroup_barrier(mem_flags::mem_threadgroup); + uint n0 = (tg*ROWS + sgid)*CQ4_NR; + if (n0 >= N) return; + uint K = num_groups*gs; + float acc[CQ4_NR]; + #pragma clang loop unroll(full) + for (uint r=0;r> 3; + #pragma clang loop unroll(full) + for (uint r=0;r>2)&3u]; w2=cb[(w>>4)&3u]; w3=cb[(w>>6)&3u]; + w4=cb[(w>>8)&3u]; w5=cb[(w>>10)&3u]; w6=cb[(w>>12)&3u]; w7=cb[(w>>14)&3u]; + w8=cb[(w>>16)&3u]; w9=cb[(w>>18)&3u]; wa=cb[(w>>20)&3u]; wb=cb[(w>>22)&3u]; + wc=cb[(w>>24)&3u]; wd=cb[(w>>26)&3u]; we=cb[(w>>28)&3u]; wf=cb[(w>>30)&3u]; + } else { + device const ushort* ps = (device const ushort*)pr; + ulong v = (ulong)ps[0] | ((ulong)ps[1] << 16) | ((ulong)ps[2] << 32); + w0=cb[v&7u]; w1=cb[(v>>3)&7u]; w2=cb[(v>>6)&7u]; w3=cb[(v>>9)&7u]; + w4=cb[(v>>12)&7u]; w5=cb[(v>>15)&7u]; w6=cb[(v>>18)&7u]; w7=cb[(v>>21)&7u]; + w8=cb[(v>>24)&7u]; w9=cb[(v>>27)&7u]; wa=cb[(v>>30)&7u]; wb=cb[(v>>33)&7u]; + wc=cb[(v>>36)&7u]; wd=cb[(v>>39)&7u]; we=cb[(v>>42)&7u]; wf=cb[(v>>45)&7u]; + } + float pacc = (float)c0.x*w0 + (float)c0.y*w1 + (float)c0.z*w2 + (float)c0.w*w3 + + (float)c1.x*w4 + (float)c1.y*w5 + (float)c1.z*w6 + (float)c1.w*w7 + + (float)c2.x*w8 + (float)c2.y*w9 + (float)c2.z*wa + (float)c2.w*wb + + (float)c3.x*wc + (float)c3.y*wd + (float)c3.z*we + (float)c3.w*wf; + acc[r] += (float)norms[(size_t)n*num_groups+g]*pacc; + } + } + #pragma clang loop unroll(full) + for (uint r=0;r= N*num_groups) return; + uint n = gid / num_groups, g = gid % num_groups; + uint K = num_groups*gs; + float nrm = (float)norms[(size_t)n*num_groups + g]; + device const uchar* pr = packed + ((size_t)n*num_groups + g)*pgb; + device half* wr = wf16 + (size_t)n*K + (size_t)g*gs; + for (uint e=0; e= N) return; + float acc = 0.f; + for (uint g = 0u; g < num_groups; ++g) { + uchar b = packed[((size_t)n*num_groups + g)*32u + lane]; + device const char* a = act_i8 + (size_t)g*128u + lane*4u; + int partial = (int)a[0]*cb[b & 3u] + (int)a[1]*cb[(b >> 2) & 3u] + + (int)a[2]*cb[(b >> 4) & 3u] + (int)a[3]*cb[(b >> 6) & 3u]; + int idot = simd_sum(partial); + if (lane == 0u) { + float ns = (float)norms[(size_t)n*num_groups + g] * cb_scale; + float sc = ns * act_scales[g]; + acc = fma((float)idot, sc, acc); + } + } + if (lane == 0u) y[n] = (half)acc; +} + kernel void cq4_transform_m( device const half* x [[buffer(0)]], device const half* recip [[buffer(1)]], @@ -684,14 +837,15 @@ kernel void emb_ortho_wide(device const uchar* packed [[buffer(0)]], device half* out [[buffer(5)]], constant uint& K [[buffer(6)]], constant uint& row [[buffer(7)]], constant float& oscale [[buffer(8)]], + constant uint& bits [[buffer(9)]], uint tg [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], uint sg [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], threadgroup float* dq [[threadgroup(0)]]) { - uint pgb = K/2u; + uint pgb = (K*bits + 7u)/8u; device const uchar* prow = packed + (size_t)row*pgb; - for (uint i=t; i>1]; uint idx=(i&1u)?(uint)(b>>4):(uint)(b&0xF); dq[i]=(float)codebook[idx]; } + for (uint i=t; i>1]; uint idx=(k&1u)?(uint)(b>>4):(uint)(b&0xF); - uint dst=perm[k]; z[dst]=(float)codebook[idx]*(float)rsign[dst]; + uint dst=perm[k]; z[dst]=(float)codebook[cq_idx(pg, k, bits)]*(float)rsign[dst]; } threadgroup_barrier(mem_flags::mem_threadgroup); for (uint h=1; h>1)]; uint idx=(ki&1u)?(uint)(b>>4):(uint)(b&0xF); v=codebook[idx]; } + if (mm>1]; uint idx=(k&1u)?(uint)(b>>4):(uint)(b&0xF); uint dst=perm[k]; z[dst]=(float)codebook[idx]*(float)rsign[dst]; } + for (uint k=t;k psoTsimd = nil; id psoTbatch = nil; id psoMega = nil, psoSwiT = nil, psoCat = nil; - id psoG = nil, psoGmr = nil; + id psoG = nil, psoGmr = nil, psoGmrLow = nil, psoDeqLow = nil; + id psoActQ = nil, psoG2i8 = nil; id psoTm = nil, psoGm = nil; id psoGmma = nil, psoGdense = nil, psoDeq = nil; id psoRotate = nil, psoRotW = nil, psoEmbOW = nil, psoRmsAddRms = nil; @@ -72,6 +73,8 @@ psoMega=pso("cq4_gemv_mega"); psoSwiT=pso("cq4_swiglu_transform"); psoCat=pso("cq4_gemv_cat"); psoTm=pso("cq4_transform_m"); psoGm=pso("cq4_gemm"); psoGmma=pso("cq4_gemm_mma"); psoGdense=pso("cq4_gemm_dense_f16"); psoDeq=pso("cq4_dequant_f16"); + psoGmrLow=pso("cq_gemv_mr_lowbit"); psoDeqLow=pso("cq_dequant_lowbit_f16"); + psoActQ=pso("cq_act_quant_i8"); psoG2i8=pso("cq2_gemv_i8"); psoRotate=pso("lmhead_rotate"); psoRotW=pso("lmhead_rotate_wide"); psoEmbOW=pso("emb_ortho_wide"); psoRmsAddRms=pso("rms_norm_add_rms_f16"); psoEmbO=pso("emb_ortho"); psoEmbH=pso("emb_hadamard"); psoEmbOm=pso("emb_ortho_m"); psoEmbHm=pso("emb_hadamard_m"); @@ -106,6 +109,8 @@ struct ResW { id packed=nil, norms=nil, codebook=nil, lsign=nil, rsign=nil, perm=nil, recip=nil; id rotation=nil; + id cb_i8=nil; + float cb_scale=1.f; id wf16=nil; }; std::unordered_map g_resident; @@ -135,23 +140,72 @@ void deinterleave_4row(const CactusQuantMatrix* W, } } +void deinterleave_4row_lowbit(const CactusQuantMatrix* W, + uint8_t* packed_nat, __fp16* norms_nat) { + const uint32_t N=W->N, ng=W->num_groups, gs=W->group_size, bits=W->bits; + const uint32_t pgb=(gs*bits+7u)/8u; + const uint32_t panel_bytes=gs*4u*bits/8u; + const uint32_t NB=N/4u; + std::memset(packed_nat, 0, (size_t)N*ng*pgb); + const uint8_t* il=W->packed_indices; + for (uint32_t nb=0; nbnorms[((size_t)nb*ng+g)*4u + r]; + uint8_t* row = packed_nat + ((size_t)n*ng+g)*pgb; + for (uint32_t k=0; k>4)*16u; + idx = (ch[((k&15u)>>2)*4u + r] >> (2u*(k&3u))) & 3u; + } else { + const uint8_t* ch = panel + (k>>2)*6u; + uint64_t word=0; std::memcpy(&word, ch, 6); + idx = (uint32_t)((word >> (r*12u + (k&3u)*3u)) & 7u); + } + uint32_t bp = k*bits; + row[bp>>3] |= (uint8_t)(idx << (bp&7u)); + if (bits == 3u && (bp&7u) > 5u) row[(bp>>3)+1u] |= (uint8_t)(idx >> (8u-(bp&7u))); + } + } + } +} + ResW& resident(const CactusQuantMatrix* W) { std::lock_guard lk(g_resident_mu); auto it = g_resident.find(W->packed_indices); if (it != g_resident.end()) return it->second; - const uint32_t K=W->K, N=W->N, ng=W->num_groups, gs=W->group_size, pgb=(gs*4u+7u)/8u; + const uint32_t K=W->K, N=W->N, ng=W->num_groups, gs=W->group_size, bits=W->bits; + const uint32_t pgb=(gs*bits+7u)/8u; ResW r; - r.codebook = buf(W->codebook, 16*sizeof(__fp16)); + r.codebook = buf(W->codebook, (size_t)(1u<left_signs, gs); r.rsign = buf(W->right_signs, gs); r.perm = buf(W->permutation, (size_t)gs*sizeof(uint32_t)); r.recip = buf(W->input_scale_recip, (size_t)K*sizeof(__fp16)); if (W->rotation) r.rotation = buf(W->rotation, (size_t)K*K*sizeof(__fp16)); + if (bits == 2u) { + int8_t cbq[4]; + float max_abs = 0.f; + for (uint32_t i = 0; i < 4u; i++) { + float v = std::fabs((float)W->codebook[i]); + if (v > max_abs) max_abs = v; + } + float scale = max_abs / 127.f; + if (scale < 1e-10f) scale = 1e-10f; + float inv = 1.f / scale; + for (uint32_t i = 0; i < 4u; i++) + cbq[i] = (int8_t)std::round((float)W->codebook[i] * inv); + r.cb_i8 = buf(cbq, 4); + r.cb_scale = scale; + } if (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) { size_t pkb=(size_t)N*ng*pgb, nmb=(size_t)N*ng*sizeof(__fp16); r.packed = [ctx().dev newBufferWithLength:pkb options:MTLResourceStorageModeShared]; r.norms = [ctx().dev newBufferWithLength:nmb options:MTLResourceStorageModeShared]; - deinterleave_4row(W, (uint8_t*)r.packed.contents, (__fp16*)r.norms.contents); + if (bits == 4u) deinterleave_4row(W, (uint8_t*)r.packed.contents, (__fp16*)r.norms.contents); + else deinterleave_4row_lowbit(W, (uint8_t*)r.packed.contents, (__fp16*)r.norms.contents); } else { r.packed = buf(W->packed_indices, (size_t)N*ng*pgb); r.norms = buf(W->norms, (size_t)N*ng*sizeof(__fp16)); @@ -617,6 +671,13 @@ static inline bool quant_fast_eligible(const CactusQuantMatrix* W) { W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; } +static inline bool quant_lowbit_eligible(const CactusQuantMatrix* W) { + return (W->bits == 2 || W->bits == 3) && W->group_size == 128 && (W->N % 16) == 0 && + !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && + W->input_scale_recip && W->left_signs && W->right_signs && W->permutation && + W->codebook && W->norms && W->packed_indices; +} + bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W) { const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; if (ctx().ok && (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->rotation && ng==1 && gs==K && (N%4)==0) { @@ -624,8 +685,10 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu if (ortho_code_k < K) { ortho_code = cactus_metal_alloc_shared((size_t)K*sizeof(__fp16)); ortho_code_k = K; } if (ortho_code) return cactus_metal_encode_quant_matmul_ortho(out, lhs, ortho_code, W); } - bool fast = ctx().ok && quant_fast_eligible(W); - if (!fast) return false; + const bool fast = ctx().ok && quant_fast_eligible(W); + const bool lowbit = !fast && ctx().ok && ctx().psoGmrLow && quant_lowbit_eligible(W); + if (!fast && !lowbit) return false; + const uint32_t bits = W->bits, pgbw = (gs*bits+7u)/8u; ResW& rw = resident(W); size_t code_bytes = (size_t)ng*gs*sizeof(__fp16); @@ -641,16 +704,37 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; hard_barrier(); + if (lowbit && bits == 2u && gs == 128u && ctx().psoActQ && ctx().psoG2i8 && rw.cb_i8) { + id actq = recycled((size_t)ng*gs); + id ascl = recycled((size_t)ng*sizeof(float)); + [g_enc setComputePipelineState:ctx().psoActQ]; + [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; + [g_enc setBuffer:actq offset:0 atIndex:1]; [g_enc setBuffer:ascl offset:0 atIndex:2]; + [g_enc setBytes:&gs length:4 atIndex:3]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + hard_barrier(); + [g_enc setComputePipelineState:ctx().psoG2i8]; + [g_enc setBuffer:actq offset:0 atIndex:0]; [g_enc setBuffer:ascl offset:0 atIndex:1]; + [g_enc setBuffer:rw.packed offset:0 atIndex:2]; [g_enc setBuffer:rw.cb_i8 offset:0 atIndex:3]; + [g_enc setBuffer:rw.norms offset:0 atIndex:4]; + setBufAt(out, (size_t)N*2, 5); + [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&N length:4 atIndex:7]; + [g_enc setBytes:&rw.cb_scale length:4 atIndex:8]; + if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake((N+7u)/8u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + barrier(); + return true; + } bool umr = g_gemv_mr && ctx().psoGmr && (N % g_mr_rows == 0u); - [g_enc setComputePipelineState:(umr?ctx().psoGmr:ctx().psoG)]; + [g_enc setComputePipelineState:(lowbit?ctx().psoGmrLow:(umr?ctx().psoGmr:ctx().psoG))]; [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:0 atIndex:1]; [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; setBufAt(out, (size_t)N*2, 4); [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; - [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; + [g_enc setBytes:&pgbw length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; + if (lowbit) [g_enc setBytes:&bits length:4 atIndex:9]; uint32_t ROWS=8; g_op="matmul"; - uint32_t grid = umr ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; + uint32_t grid = (lowbit || umr) ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; barrier(); return true; @@ -805,16 +889,19 @@ bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { if (!ctx().ok) return false; - if (!quant_fast_eligible(W)) return false; + if (!quant_fast_eligible(W) && !quant_lowbit_eligible(W)) return false; resident(W); return true; } bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M) { if (M == 1) return cactus_metal_encode_quant_matmul(out, lhs, W); - const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; - bool fast = ctx().ok && quant_fast_eligible(W); - if (!fast) return false; + const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; + const uint32_t bits=W->bits, pgb=(gs*bits+7u)/8u; + const bool fast = ctx().ok && quant_fast_eligible(W); + const bool lowbit = !fast && ctx().ok && quant_lowbit_eligible(W) && + g_predequant && ctx().psoGdense && ctx().psoDeqLow; + if (!fast && !lowbit) return false; ResW& rw = resident(W); size_t code_bytes = (size_t)M*ng*gs*sizeof(__fp16); if (!g_code_buf_m || (size_t)g_code_buf_m.length < code_bytes) @@ -832,14 +919,15 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus if (!g_skip_transform) [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; hard_barrier(); - if (g_predequant && ctx().psoGdense && ctx().psoDeq) { + if ((g_predequant && ctx().psoGdense && ctx().psoDeq) || lowbit) { if (!rw.wf16) { rw.wf16 = [ctx().dev newBufferWithLength:(size_t)N*K*2 options:MTLResourceStorageModeShared]; - [g_enc setComputePipelineState:ctx().psoDeq]; + [g_enc setComputePipelineState:(lowbit?ctx().psoDeqLow:ctx().psoDeq)]; [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.wf16 offset:0 atIndex:3]; [g_enc setBytes:&gs length:4 atIndex:4]; [g_enc setBytes:&ng length:4 atIndex:5]; [g_enc setBytes:&pgb length:4 atIndex:6]; [g_enc setBytes:&N length:4 atIndex:7]; + if (lowbit) [g_enc setBytes:&bits length:4 atIndex:8]; [g_enc dispatchThreads:MTLSizeMake((size_t)N*ng,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; hard_barrier(); } @@ -873,8 +961,12 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* code, const CactusQuantMatrix* W) { - if (!ctx().ok || !W->rotation || W->bits != 4) return false; - const uint32_t K=W->K, N=W->N, ng=W->num_groups, gs=W->group_size, pgb=(gs*4u+7u)/8u; + if (!ctx().ok || !W->rotation) return false; + const uint32_t K=W->K, N=W->N, ng=W->num_groups, gs=W->group_size, bits=W->bits; + const uint32_t pgb=(gs*bits+7u)/8u; + const bool lowbit = (bits == 2 || bits == 3); + if (bits != 4 && !lowbit) return false; + if (lowbit && (!ctx().psoGmrLow || (N % 16) != 0)) return false; if (ng != 1 || gs != K || (N % 4) != 0) return false; ResW& rw = resident(W); if (!rw.rotation || !rw.packed) return false; @@ -897,23 +989,25 @@ bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* co hard_barrier(); bool umr = g_gemv_mr && ctx().psoGmr && (N % g_mr_rows == 0u); - [g_enc setComputePipelineState:(umr?ctx().psoGmr:ctx().psoG)]; + [g_enc setComputePipelineState:(lowbit?ctx().psoGmrLow:(umr?ctx().psoGmr:ctx().psoG))]; setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:0 atIndex:1]; [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; setBufAt(out, (size_t)N*2, 4); [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; + if (lowbit) [g_enc setBytes:&bits length:4 atIndex:9]; uint32_t ROWS=8; - uint32_t grid = umr ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; + uint32_t grid = (lowbit || umr) ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; barrier(); return true; } bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W, float scale) { - if (!ctx().ok || !W->rotation || W->bits != 4) return false; - const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size; + if (!ctx().ok || !W->rotation || (W->bits != 4 && W->bits != 2 && W->bits != 3)) return false; + const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size, bits=W->bits; if (ng != 1 || gs != K) return false; + if (bits != 4 && !(ctx().psoEmbOW && (K % 8u) == 0u)) return false; ResW& rw = resident(W); if (!rw.packed || !rw.rotation || !rw.codebook || !rw.norms || !rw.recip) return false; ensureEncoder(); @@ -924,7 +1018,7 @@ bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQu [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.recip offset:0 atIndex:3]; [g_enc setBuffer:rw.rotation offset:0 atIndex:4]; setBufAt(out, (size_t)K*2, 5); [g_enc setBytes:&K length:4 atIndex:6]; [g_enc setBytes:&row length:4 atIndex:7]; - [g_enc setBytes:&scale length:4 atIndex:8]; + [g_enc setBytes:&scale length:4 atIndex:8]; [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc setThreadgroupMemoryLength:(size_t)K*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(K/8u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; barrier(); @@ -948,7 +1042,7 @@ bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQu if (it != g_resident_emb.end()) return it->second; const uint32_t K=W->K, gs=W->group_size; ResW r; - r.codebook = buf(W->codebook, 16*sizeof(__fp16)); + r.codebook = buf(W->codebook, (size_t)(1u<bits)*sizeof(__fp16)); r.lsign = buf(W->left_signs, gs); r.rsign = buf(W->right_signs, gs); r.perm = buf(W->permutation, (size_t)gs*sizeof(uint32_t)); @@ -957,8 +1051,9 @@ bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQu } bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const CactusQuantMatrix* W) { - if (!ctx().ok || W->bits != 4 || (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) return false; - const uint32_t K=W->K, gs=W->group_size, ng=W->num_groups, pgb=(gs*4u+7u)/8u; + if (!ctx().ok || (W->bits != 4 && W->bits != 2 && W->bits != 3) || + (W->flags & (CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW))) return false; + const uint32_t K=W->K, gs=W->group_size, ng=W->num_groups, bits=W->bits, pgb=(gs*bits+7u)/8u; if (gs > 256 || (gs & (gs-1)) != 0 || !W->packed_indices || !W->norms || !W->codebook || !W->left_signs || !W->right_signs || !W->permutation || !W->input_scale_recip) return false; ResW& rm = resident_emb_meta(W); @@ -974,7 +1069,7 @@ bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const Cactu [g_enc setBuffer:nrow offset:0 atIndex:2]; [g_enc setBuffer:rm.recip offset:0 atIndex:3]; [g_enc setBuffer:rm.lsign offset:0 atIndex:4]; [g_enc setBuffer:rm.rsign offset:0 atIndex:5]; [g_enc setBuffer:rm.perm offset:0 atIndex:6]; setBufAt(out, (size_t)K*2, 7); - [g_enc setBytes:&gs length:4 atIndex:8]; + [g_enc setBytes:&gs length:4 atIndex:8]; [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc setThreadgroupMemoryLength:(size_t)gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(gs,1,1)]; barrier(); @@ -994,8 +1089,8 @@ bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const Cactu } bool cactus_metal_encode_embedding_ortho_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M) { - if (!ctx().ok || !ctx().psoEmbOm || !W->rotation || W->bits != 4) return false; - const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size; + if (!ctx().ok || !ctx().psoEmbOm || !W->rotation || (W->bits != 4 && W->bits != 2 && W->bits != 3)) return false; + const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size, bits=W->bits; if (ng != 1 || gs != K) return false; ResW& rw = resident(W); if (!rw.packed || !rw.rotation || !rw.codebook || !rw.norms || !rw.recip) return false; @@ -1007,14 +1102,16 @@ bool cactus_metal_encode_embedding_ortho_m(void* out, const CactusQuantMatrix* W [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.recip offset:0 atIndex:3]; [g_enc setBuffer:rw.rotation offset:0 atIndex:4]; [g_enc setBuffer:rb offset:0 atIndex:5]; setBufAt(out, (size_t)M*K*2, 6); [g_enc setBytes:&K length:4 atIndex:7]; [g_enc setBytes:&M length:4 atIndex:8]; + [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc dispatchThreadgroups:MTLSizeMake(((K+15)/16)*((M+15)/16),1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; barrier(); return true; } bool cactus_metal_encode_embedding_hadamard_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M) { - if (!ctx().ok || !ctx().psoEmbHm || W->bits != 4 || (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) return false; - const uint32_t K=W->K, gs=W->group_size, ng=W->num_groups, pgb=(gs*4u+7u)/8u; + if (!ctx().ok || !ctx().psoEmbHm || (W->bits != 4 && W->bits != 2 && W->bits != 3) || + (W->flags & (CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW))) return false; + const uint32_t K=W->K, gs=W->group_size, ng=W->num_groups, bits=W->bits, pgb=(gs*bits+7u)/8u; if (gs > 256 || (gs & (gs-1)) != 0 || !W->packed_indices || !W->norms || !W->codebook || !W->left_signs || !W->right_signs || !W->permutation || !W->input_scale_recip) return false; ResW& rm = resident_emb_meta(W); @@ -1033,6 +1130,7 @@ bool cactus_metal_encode_embedding_hadamard_m(void* out, const CactusQuantMatrix [g_enc setBuffer:rm.lsign offset:0 atIndex:4]; [g_enc setBuffer:rm.rsign offset:0 atIndex:5]; [g_enc setBuffer:rm.perm offset:0 atIndex:6]; setBufAt(out, (size_t)M*K*2, 7); [g_enc setBytes:&gs length:4 atIndex:8]; [g_enc setBytes:&ng length:4 atIndex:9]; [g_enc setBytes:&K length:4 atIndex:10]; + [g_enc setBytes:&bits length:4 atIndex:11]; [g_enc setThreadgroupMemoryLength:(size_t)gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs,1,1)]; barrier(); From 4a2efc4767e412ca24a370189829542bba200852 Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 06:12:56 +0200 Subject: [PATCH 11/36] cleanup Signed-off-by: jakmro --- README.md | 1 - cactus-engine/src/model.cpp | 6 +- cactus-graph/cactus_graph.h | 1 - cactus-graph/src/execute.cpp | 31 +- cactus-graph/src/gpu_fused.cpp | 327 +++------------------- cactus-graph/src/ops_nn.cpp | 8 +- cactus-kernels/CMakeLists.txt | 4 +- cactus-kernels/metal_backend.h | 15 +- cactus-kernels/src/cactus_kernels.metal | 109 ++------ cactus-kernels/src/metal_backend.mm | 279 +++--------------- cactus-kernels/src/metal_backend_stub.cpp | 10 +- python/cactus/cli/__init__.py | 2 - python/cactus/cli/serve.py | 6 + 13 files changed, 121 insertions(+), 678 deletions(-) diff --git a/README.md b/README.md index bf37101a4..b44e39712 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,6 @@ graph.hard_reset(); │ --language language code (default: en) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ │ --platform auto|cpu|apple target platform (default: auto) │ -│ --backend auto|cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ │ diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 34e49cc4d..07c1efb26 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -603,7 +603,7 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, } if (decoder_prefill_ && decoder_prefill_->graph - && std::getenv("CACTUS_NO_PREFILL_WARM") == nullptr) { + ) { try { if (prefill_encoder_ && prefill_encoder_->graph) { for (auto& b : prefill_encoder_->input_buffers) std::fill(b.begin(), b.end(), 0); @@ -1303,9 +1303,7 @@ void Model::execute_prefill_chunk(Component& chunk_comp, Component* enc_comp, si copy_component_outputs_to_chunk_inputs(*encoder_, chunk_comp, i); } } - static const bool fused_prefill = std::getenv("CACTUS_FUSED_PREFILL") != nullptr; - if (!fused_prefill || !chunk_comp.graph->execute_gpu_fused_prefill(static_cast(chunk_tokens))) - chunk_comp.graph->execute(); + chunk_comp.graph->execute(); } void Model::reset_prefill_stats() { diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 795b8678b..80a318605 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -729,7 +729,6 @@ class CactusGraph { void execute(const std::string& profile_file = ""); bool execute_gpu_fused(); - bool execute_gpu_fused_prefill(uint32_t M); bool extract_ple_pathway(FusedEmbedCtx& ctx) const; void hard_reset(); void soft_reset(); diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index eb5f56300..cc43d6adc 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -846,7 +846,6 @@ void CactusGraph::execute(const std::string& profile_file) { } return false; }; - auto may_alias_input = aliases_input; auto preallocates_output = [&](const GraphNode& node) { return !aliases_input(node); }; std::vector last_use(n, 0); @@ -864,7 +863,7 @@ void CactusGraph::execute(const std::string& profile_file) { std::vector keep_until_graph_cleanup(n, false); for (size_t i = 0; i < n; ++i) { const auto& node = *nodes_[i]; - if (!may_alias_input(node) || node.input_ids.empty()) continue; + if (!aliases_input(node) || node.input_ids.empty()) continue; auto it = node_index_map_.find(node.input_ids[0]); if (it == node_index_map_.end()) continue; size_t base_idx = it->second; @@ -888,32 +887,6 @@ void CactusGraph::execute(const std::string& profile_file) { GraphNode& nd = *nodes_[idx]; if (aliases_input(nd)) nd.output_buffer.external_data = nullptr; }; - const bool concurrent = cactus_metal_concurrent(); - struct Acc { uintptr_t lo, hi; bool w; }; - std::vector accessed; - auto conflicts = [&](uintptr_t lo, uintptr_t hi, bool w) { - for (auto& a : accessed) if ((w || a.w) && lo < a.hi && a.lo < hi) return true; - return false; - }; - auto hazard_barrier = [&](GraphNode& nd) { - if (!concurrent) return; - const bool allW = (nd.op_type == OpType::KV_CACHE_APPEND); - std::vector mine; bool hit = false; - for (size_t j = 0; j < nd.input_ids.size(); ++j) { - const auto& ib = get_input(nd, j, nodes_, node_index_map_); - if (!ib.get_data()) continue; - uintptr_t lo = (uintptr_t)ib.get_data(), hi = lo + ib.byte_size; - if (conflicts(lo, hi, allW)) hit = true; - mine.push_back({lo, hi, allW}); - } - if (nd.output_buffer.get_data()) { - uintptr_t lo = (uintptr_t)nd.output_buffer.get_data(), hi = lo + nd.output_buffer.byte_size; - if (conflicts(lo, hi, true)) hit = true; - mine.push_back({lo, hi, true}); - } - if (hit) { cactus_metal_barrier(); accessed.clear(); } - for (auto& m : mine) accessed.push_back(m); - }; for (size_t i = 0; i < n; ++i) { auto& node = nodes_[i]; const OpType ot = node->op_type; @@ -944,11 +917,9 @@ void CactusGraph::execute(const std::string& profile_file) { if (p) { node->output_buffer.release_to_pool(pool); node->output_buffer.set_external(p); } else node->output_buffer.resize_from_pool(pool); } - hazard_barrier(*node); bool encoded = try_encode_gpu(*node, nodes_, node_index_map_); if (!encoded) { cactus_metal_session_sync(); - accessed.clear(); dispatch_node(*node, nodes_, node_index_map_); } if (ot == OpType::PERSISTENT) populated_node_ids_.insert(node->id); diff --git a/cactus-graph/src/gpu_fused.cpp b/cactus-graph/src/gpu_fused.cpp index a0d6470ad..0192fdeb4 100644 --- a/cactus-graph/src/gpu_fused.cpp +++ b/cactus-graph/src/gpu_fused.cpp @@ -1,5 +1,3 @@ - - #include "../cactus_graph.h" #include "cactus_kernels.h" #include "metal_backend.h" @@ -8,28 +6,13 @@ #include #include #include -#include namespace { constexpr int NL = 35, HID = 1536, NQH = 8, PLE_DIM = 256; constexpr float RMS_EPS = 1e-6f, GATE_SCALE = 0.015625f, SOFTCAP = 30.0f; -static const bool g_fuse_nr = (std::getenv("CACTUS_NO_FUSE_NR") == nullptr); -static const unsigned g_mega_mask = [](){ - const char* e = std::getenv("CACTUS_MEGA"); - if (std::getenv("CACTUS_NO_MEGA")) return 0u; - return e ? (unsigned)std::strtoul(e, nullptr, 0) : 0x38u; -}(); -static const bool g_mega_qkv = (g_mega_mask & 1u), g_mega_o = (g_mega_mask & 2u), - g_mega_gu = (g_mega_mask & 4u), g_mega_ple = (g_mega_mask & 8u), - g_mega_pj = (g_mega_mask & 16u), g_mega_dswt = (g_mega_mask & 32u), - g_mega_down = (g_mega_mask & 64u); -static const bool g_fattn = (std::getenv("CACTUS_NO_FATTN") == nullptr); -static const int g_flush_layers = [](){ - const char* e = std::getenv("CACTUS_FLUSH_LAYERS"); - return e ? std::atoi(e) : 12; -}(); +constexpr int FLUSH_LAYERS = 3; struct LayerW { CactusQuantMatrix q, o, ple_down, ple_up, gate, up, down; @@ -225,7 +208,6 @@ bool CactusGraph::execute_gpu_fused() { } cactus_metal_session_begin(); - cactus_metal_session_set_concurrent(true); auto encode_step = [&](int pos_, uint32_t tok, void** lg_out) -> int { G.arena.reset(); @@ -236,15 +218,12 @@ bool CactusGraph::execute_gpu_fused() { if (fold) { const int PK = (int)g_fe.proj.N; void* pe = fresh(g_fe.ple.K); - cactus_metal_group_begin(); bool e1 = cactus_metal_encode_embedding_ortho(h, tok, &G.lm_head, g_fe.emb_scale); bool e2 = cactus_metal_encode_embedding_hadamard(pe, tok, &g_fe.ple); - cactus_metal_group_end(); if (!e1 || !e2) return 0; void* pa = fresh(PK); cactus_metal_encode_scalar(2, pa, pe, PK, g_fe.ple_scale); void* pj = fresh(PK); - const CactusQuantMatrix* Wpj[1]={&g_fe.proj}; void* ypj[1]={pj}; - if (!(g_mega_pj && cactus_metal_encode_gemv_mega(ypj, h, nullptr, 0.f, Wpj, 1, nullptr)) + if (!cactus_metal_encode_transform_gemv(pj, h, &g_fe.proj, nullptr) && !cactus_metal_encode_quant_matmul(pj, h, &g_fe.proj)) return 0; void* pjs= fresh(PK); cactus_metal_encode_scalar(2, pjs, pj, PK, g_fe.proj_scale); void* plt= fresh(PK); @@ -267,7 +246,7 @@ bool CactusGraph::execute_gpu_fused() { for (int l = 0; l < NL; ++l) { - if (g_flush_layers > 0 && l > 0 && (l % g_flush_layers) == 0) cactus_metal_session_flush(); + if (l > 0 && (l % FLUSH_LAYERS) == 0) cactus_metal_session_flush(); LayerW& w = G.L[l]; const int hd = w.hd, QD = NQH*hd; const void* rc = w.global ? G.cos_g : G.cos_s; @@ -287,31 +266,25 @@ bool CactusGraph::execute_gpu_fused() { void* v = fresh(hd); const CactusQuantMatrix* Wqkv[3]={&w.q,&w.k,&w.v}; void* yqkv[3]={q,k,v}; - bool qkv_mega = g_mega_qkv && cactus_metal_encode_gemv_mega(yqkv, xn, nullptr, 0.f, Wqkv, 3, nullptr); - if (!qkv_mega) { - void *cq=fresh(HID), *ck=fresh(HID), *cv=fresh(HID); - void* codes[3]={cq,ck,cv}; - bool qkv_batched = cactus_metal_encode_transform_batch(xn, Wqkv, 3, codes); - if (qkv_batched) { - const void* ccodes[3]={cq,ck,cv}; - if (!cactus_metal_encode_gemv_cat(yqkv, ccodes, Wqkv, 3)) { - cactus_metal_group_begin(); - cactus_metal_encode_gemv_precoded(q, cq, &w.q); - cactus_metal_encode_gemv_precoded(k, ck, &w.k); - cactus_metal_encode_gemv_precoded(v, cv, &w.v); - cactus_metal_group_end(); - } - } else { - cactus_metal_encode_quant_matmul(q, xn, &w.q); - cactus_metal_encode_quant_matmul(k, xn, &w.k); - cactus_metal_encode_quant_matmul(v, xn, &w.v); + void *cq=fresh(HID), *ck=fresh(HID), *cv=fresh(HID); + void* codes[3]={cq,ck,cv}; + if (cactus_metal_encode_transform_batch(xn, Wqkv, 3, codes)) { + const void* ccodes[3]={cq,ck,cv}; + if (!cactus_metal_encode_gemv_cat(yqkv, ccodes, Wqkv, 3)) { + cactus_metal_encode_gemv_precoded(q, cq, &w.q); + cactus_metal_encode_gemv_precoded(k, ck, &w.k); + cactus_metal_encode_gemv_precoded(v, cv, &w.v); } + } else { + cactus_metal_encode_quant_matmul(q, xn, &w.q); + cactus_metal_encode_quant_matmul(k, xn, &w.k); + cactus_metal_encode_quant_matmul(v, xn, &w.v); } size_t clen = km[0]; bool wrap = (!w.global && clen >= (size_t)Wn); size_t slot = wrap ? (size_t)(Sn + ((clen - Sn) % Rn)) : clen; size_t f_kv_end = wrap ? (size_t)Wn : std::min(clen + 1, (size_t)pos_ + 1); - fattn = g_fattn && cactus_metal_encode_attention_fused_i8(attn, q, k, v, + fattn = cactus_metal_encode_attention_fused_i8(attn, q, k, v, (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, w.q_norm, w.k_norm, w.global ? G.vnorm_g : G.vnorm, rc, rs, NQH, (uint32_t)hd, (uint32_t)hd, @@ -323,14 +296,9 @@ bool CactusGraph::execute_gpu_fused() { void* qr = fresh(QD); void* kr = fresh(hd); void* vn = fresh(hd); - bool okq = false, okk = false; - cactus_metal_group_begin(); - if (g_fuse_nr) { - okq = cactus_metal_encode_rms_norm_rope(qr, q, w.q_norm, rc, rs, NQH, hd, RMS_EPS); - okk = cactus_metal_encode_rms_norm_rope(kr, k, w.k_norm, rc, rs, 1, hd, RMS_EPS); - } + bool okq = cactus_metal_encode_rms_norm_rope(qr, q, w.q_norm, rc, rs, NQH, hd, RMS_EPS); + bool okk = cactus_metal_encode_rms_norm_rope(kr, k, w.k_norm, rc, rs, 1, hd, RMS_EPS); cactus_metal_encode_rms_norm(vn, v, w.global ? G.vnorm_g : G.vnorm, 1, hd, RMS_EPS); - cactus_metal_group_end(); if (!okq) { void* qn = fresh(QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, NQH, hd, RMS_EPS); cactus_metal_encode_rope(qr, qn, rc, rs, NQH, hd); @@ -339,12 +307,10 @@ bool CactusGraph::execute_gpu_fused() { void* kn = fresh(hd); cactus_metal_encode_rms_norm(kn, k, w.k_norm, 1, hd, RMS_EPS); cactus_metal_encode_rope(kr, kn, rc, rs, 1, hd); } - cactus_metal_group_begin(); cactus_metal_encode_kv_append_i8(kr, (char*)w.kc+64, (char*)w.kc+64+mx*hd, 1, hd, (uint32_t)slot, 32, hd*2, mx*hd, mx*ng*sizeof(float)); cactus_metal_encode_kv_append_i8(vn, (char*)w.vc+64, (char*)w.vc+64+mx*hd, 1, hd, (uint32_t)slot, 32, hd*2, mx*hd, mx*ng*sizeof(float)); - cactus_metal_group_end(); km[0] = clen + 1; vm[0] = clen + 1; size_t hist, total, kv_end; const void *knewp, *vnewp; if (wrap) { @@ -359,14 +325,10 @@ bool CactusGraph::execute_gpu_fused() { hist*hd, hist*hd, hist*ng*sizeof(float), hist*ng*sizeof(float)); } } else { - const CactusQuantMatrix* Wq1[1]={&w.q}; - void* yq1[1]={q}; - if (!g_mega_qkv || !cactus_metal_encode_gemv_mega(yq1, xn, nullptr, 0.f, Wq1, 1, nullptr)) { - cactus_metal_encode_quant_matmul(q, xn, &w.q); - } + cactus_metal_encode_quant_matmul(q, xn, &w.q); size_t clen = km[0]; size_t s_kv_end = (!w.global && clen > (size_t)Wn) ? (size_t)Wn : std::min(clen, (size_t)pos_ + 1); - fattn = g_fattn && cactus_metal_encode_attention_fused_i8(attn, q, nullptr, nullptr, + fattn = cactus_metal_encode_attention_fused_i8(attn, q, nullptr, nullptr, (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, w.q_norm, nullptr, nullptr, rc, rs, NQH, (uint32_t)hd, (uint32_t)hd, @@ -374,7 +336,7 @@ bool CactusGraph::execute_gpu_fused() { mx*hd, mx*hd, mx*ng*sizeof(float), mx*ng*sizeof(float)); if (!fattn) { void* qr = fresh(QD); - if (!g_fuse_nr || !cactus_metal_encode_rms_norm_rope(qr, q, w.q_norm, rc, rs, NQH, hd, RMS_EPS)) { + if (!cactus_metal_encode_rms_norm_rope(qr, q, w.q_norm, rc, rs, NQH, hd, RMS_EPS)) { void* qn = fresh(QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, NQH, hd, RMS_EPS); cactus_metal_encode_rope(qr, qn, rc, rs, NQH, hd); } @@ -392,11 +354,7 @@ bool CactusGraph::execute_gpu_fused() { } } void* o = fresh(HID); - { - const CactusQuantMatrix* Wo[1]={&w.o}; void* yo[1]={o}; - if (!g_mega_o || !cactus_metal_encode_gemv_mega(yo, attn, nullptr, 0.f, Wo, 1, nullptr)) - cactus_metal_encode_quant_matmul(o, attn, &w.o); - } + cactus_metal_encode_quant_matmul(o, attn, &w.o); void* h1 = fresh(HID); void* xn2 = fresh(HID); if (!cactus_metal_encode_rms_norm_add_rms(h1, xn2, o, w.post_attn, h, w.pre_ffn, 1, HID, RMS_EPS, 1.0f)) { @@ -411,31 +369,23 @@ bool CactusGraph::execute_gpu_fused() { const CactusQuantMatrix* Wgu[2]={&w.gate,&w.up}; { void* ygu[2]={gate,up}; - if (!g_mega_gu || !cactus_metal_encode_gemv_mega(ygu, xn2, nullptr, 0.f, Wgu, 2, nullptr)) { - void *cg=fresh(HID), *cu=fresh(HID); void* cgu[2]={cg,cu}; - bool gu_batched = cactus_metal_encode_transform_batch(xn2, Wgu, 2, cgu); - if (gu_batched) { - const void* gcodes[2]={cg,cu}; - if (!cactus_metal_encode_gemv_cat(ygu, gcodes, Wgu, 2)) { - cactus_metal_group_begin(); - cactus_metal_encode_gemv_precoded(gate, cg, &w.gate); - cactus_metal_encode_gemv_precoded(up, cu, &w.up); - cactus_metal_group_end(); - } - } else { - cactus_metal_encode_quant_matmul(gate, xn2, &w.gate); - cactus_metal_encode_quant_matmul(up, xn2, &w.up); + void *cg=fresh(HID), *cu=fresh(HID); void* cgu[2]={cg,cu}; + if (cactus_metal_encode_transform_batch(xn2, Wgu, 2, cgu)) { + const void* gcodes[2]={cg,cu}; + if (!cactus_metal_encode_gemv_cat(ygu, gcodes, Wgu, 2)) { + cactus_metal_encode_gemv_precoded(gate, cg, &w.gate); + cactus_metal_encode_gemv_precoded(up, cu, &w.up); } + } else { + cactus_metal_encode_quant_matmul(gate, xn2, &w.gate); + cactus_metal_encode_quant_matmul(up, xn2, &w.up); } } void* mo = fresh(HID); { - const CactusQuantMatrix* Wd[1]={&w.down}; void* ymo[1]={mo}; void* gcode = fresh(M); - if (g_mega_down && cactus_metal_encode_gemv_mega_ex(ymo, gate, nullptr, 0.f, Wd, 1, up, true, GATE_SCALE)) { - } else if (g_mega_dswt && cactus_metal_encode_swiglu_transform(gcode, gate, up, &w.down, GATE_SCALE) - && cactus_metal_encode_gemv_precoded(mo, gcode, &w.down)) { - } else { + if (!cactus_metal_encode_swiglu_transform(gcode, gate, up, &w.down, GATE_SCALE) + || !cactus_metal_encode_gemv_precoded(mo, gcode, &w.down)) { void* g3 = fresh(M); cactus_metal_encode_swiglu(g3, gate, up, M, GATE_SCALE); cactus_metal_encode_quant_matmul(mo, g3, &w.down); } @@ -446,11 +396,8 @@ bool CactusGraph::execute_gpu_fused() { void* pu = fresh(HID); { const void* pls = (const char*)pleBase + (size_t)l*PLE_DIM*2; - const CactusQuantMatrix* Wpd[1]={&w.ple_down}; void* ypd[1]={pm}; - const CactusQuantMatrix* Wpu[1]={&w.ple_up}; void* ypu[1]={pu}; - if (g_mega_ple && cactus_metal_encode_gemv_mega(ypd, h, nullptr, 0.f, Wpd, 1, pls) - && cactus_metal_encode_gemv_mega(ypu, pm, nullptr, 0.f, Wpu, 1, nullptr)) { - } else { + if (!cactus_metal_encode_transform_gemv(pm, h, &w.ple_down, pls) + || !cactus_metal_encode_transform_gemv(pu, pm, &w.ple_up, nullptr)) { void* ps = fresh(PLE_DIM); cactus_metal_encode_quant_matmul(ps, h, &w.ple_down); cactus_metal_encode_swiglu(pm, ps, pls, PLE_DIM, 1.0f); cactus_metal_encode_quant_matmul(pu, pm, &w.ple_up); @@ -462,7 +409,7 @@ bool CactusGraph::execute_gpu_fused() { const void* next_w = (l+1 < NL) ? G.L[l+1].in_norm : G.final_norm; if (cactus_metal_encode_rms_norm_add_rms(h3, xnn, pu, w.post_ple, h, next_w, 1, HID, RMS_EPS, ls)) { h = h3; xn = xnn; - } else if (g_fuse_nr && cactus_metal_encode_rms_norm_add_scale(h3, pu, w.post_ple, h, 1, HID, RMS_EPS, ls)) { + } else if (cactus_metal_encode_rms_norm_add_scale(h3, pu, w.post_ple, h, 1, HID, RMS_EPS, ls)) { cactus_metal_encode_rms_norm(xnn, h3, next_w, 1, HID, RMS_EPS); h = h3; xn = xnn; } else { @@ -508,7 +455,6 @@ bool CactusGraph::execute_gpu_fused() { cactus_metal_set_active(false); if (rc == 0) { cactus_metal_session_sync(); - cactus_metal_session_set_concurrent(false); return false; } if (rc == 1) { @@ -521,213 +467,12 @@ bool CactusGraph::execute_gpu_fused() { B(3860).set_external(lg); g_last_step_adjusted = false; } - cactus_metal_session_set_concurrent(false); return true; } -namespace { -struct PreAttn { size_t window=0, v_hdim=0, po=0; bool causal=true; float scale=0.f; }; -struct GpuModelP { - bool init = false; - const void* owner = nullptr; - LayerW L[NL]; - size_t kc_idx[15], vc_idx[15]; - PreAttn A[NL]; - const void *vnorm, *vnorm_g, *cosL, *sinL, *cosG, *sinG, *final_norm; - CactusQuantMatrix lm_head; - Arena arena; -}; -static GpuModelP GP; -} - -bool CactusGraph::execute_gpu_fused_prefill(uint32_t M) { - if (!cactus_metal_available() || M <= 1) return false; - const size_t n = nodes_.size(); - if (n < 3861) return false; - auto B = [&](size_t idx) -> BufferDesc& { return nodes_[idx]->output_buffer; }; - auto P = [&](size_t idx) -> void* { return B(idx).get_data(); }; - if (B(0).total_size != (size_t)M*HID || nodes_[0]->op_type != OpType::INPUT) return false; - g_gpu_argmax_valid = false; - - if (!GP.init) { - auto cq = [&](size_t idx){ return B(idx).to_cq_matrix(); }; - for (int l = 0; l < NL; ++l) { - LayerW& w = GP.L[l]; - w.global = (l % 5 == 4); w.hd = w.global ? 512 : 256; w.shared = (l >= 15); - if (!w.shared) { - size_t b1 = 3 + 10*l, b2 = 334 + 6*l; - w.in_norm=P(b1); w.q_norm=P(b1+1); w.k_norm=P(b1+2); w.post_attn=P(b1+3); - w.pre_ffn=P(b1+4); w.gate=cq(b1+5); w.up=cq(b1+6); w.down=cq(b1+7); - w.post_ffn=P(b1+8); w.post_ple=P(b1+9); - w.q=cq(b2); w.k=cq(b2+1); w.v=cq(b2+2); w.o=cq(b2+3); w.ple_down=cq(b2+4); w.ple_up=cq(b2+5); - w.mlp = 6144; - } else { - int j = l - 15; size_t b1 = 153 + 9*j, b2 = 424 + 4*j; - w.in_norm=P(b1); w.q_norm=P(b1+1); w.post_attn=P(b1+2); w.pre_ffn=P(b1+3); - w.gate=cq(b1+4); w.up=cq(b1+5); w.down=cq(b1+6); w.post_ffn=P(b1+7); w.post_ple=P(b1+8); - w.q=cq(b2); w.o=cq(b2+1); w.ple_down=cq(b2+2); w.ple_up=cq(b2+3); - w.mlp = 12288; - } - w.scalar = P(505 + l); - } - GP.vnorm=P(540); GP.vnorm_g=P(541); GP.cosL=P(542); GP.sinL=P(543); GP.cosG=P(544); GP.sinG=P(545); - GP.final_norm=P(333); GP.lm_head=B(504).to_cq_matrix(); - std::vector cidx; - for (size_t i = 0; i < n && cidx.size() < 30; ++i) - if (nodes_[i]->op_type == OpType::KV_CACHE_STATE) cidx.push_back(i); - if (cidx.size() < 30) return false; - for (int l = 0; l < 15; ++l) { GP.kc_idx[l]=cidx[2*l]; GP.vc_idx[l]=cidx[2*l+1]; } - int ai = 0; - for (size_t i = 0; i < n && ai < NL; ++i) { - if (nodes_[i]->op_type != OpType::ATTENTION_CACHED) continue; - auto& pr = nodes_[i]->params; - GP.A[ai].window = pr.window_size; GP.A[ai].causal = pr.is_causal; - GP.A[ai].scale = pr.scale; GP.A[ai].v_hdim = pr.v_head_dim; GP.A[ai].po = pr.position_offset; - ai++; - } - if (ai < NL) return false; - GP.owner = this; - GP.init = true; - } - if (GP.owner != this) { - GP.init = false; - return false; - } - - for (int l = 0; l < 15; ++l) { GP.L[l].kc = P(GP.kc_idx[l]); GP.L[l].vc = P(GP.vc_idx[l]); } - for (int l = 15; l < NL; ++l) { int src = GP.L[l].global ? 14 : 13; GP.L[l].kc = GP.L[src].kc; GP.L[l].vc = GP.L[src].vc; } - for (int l = 0; l < NL; ++l) if (!GP.L[l].kc || !GP.L[l].vc) return false; - - int pos0 = (int)((const uint64_t*)GP.L[0].kc)[0]; - for (int l = 0; l < 15; ++l) { - const uint64_t* km = (const uint64_t*)GP.L[l].kc; - if (km[0] + M > km[1]) return false; - } - - GP.arena.reset(); - auto fresh = [&](size_t elems){ return GP.arena.fresh(elems * 2); }; - cactus_metal_session_begin(); - cactus_metal_set_active(true); - auto bail = [&](){ cactus_metal_set_active(false); cactus_metal_session_end(); return false; }; - - void* csL = fresh((size_t)M*256); void* snL = fresh((size_t)M*256); - void* csG = fresh((size_t)M*512); void* snG = fresh((size_t)M*512); - for (uint32_t m = 0; m < M; ++m) { - int p = pos0 + (int)m; - std::memcpy((char*)csL+(size_t)m*256*2, (const char*)GP.cosL+(size_t)p*256*2, 256*2); - std::memcpy((char*)snL+(size_t)m*256*2, (const char*)GP.sinL+(size_t)p*256*2, 256*2); - std::memcpy((char*)csG+(size_t)m*512*2, (const char*)GP.cosG+(size_t)p*512*2, 512*2); - std::memcpy((char*)snG+(size_t)m*512*2, (const char*)GP.sinG+(size_t)p*512*2, 512*2); - } - - void* h = fresh((size_t)M*HID); - std::memcpy(h, P(0), (size_t)M*HID*2); - const void* pleBase = P(1); - - for (int l = 0; l < NL; ++l) { - LayerW& w = GP.L[l]; - const int hd = w.hd, QD = NQH*hd; - const void* rc = w.global ? csG : csL; - const void* rs = w.global ? snG : snL; - uint64_t* km = (uint64_t*)w.kc; - size_t max_seq = km[1], kv_heads = km[2], sink = km[4]; - size_t ngK = (hd+31)/32; - PreAttn& AP = GP.A[l]; - float ascale = AP.scale != 0.f ? AP.scale : 1.0f/std::sqrt((float)hd); - - void* xn = fresh((size_t)M*HID); cactus_metal_encode_rms_norm(xn, h, w.in_norm, M, HID, RMS_EPS); - void* q = fresh((size_t)M*QD); if(!cactus_metal_encode_quant_matmul_m(q, xn, &w.q, M)) return bail(); - void* qn = fresh((size_t)M*QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, (size_t)M*NQH, hd, RMS_EPS); - void* qr = fresh((size_t)M*QD); cactus_metal_encode_rope_m(qr, qn, rc, rs, NQH, hd, M); - - size_t hist, newlen; const void *knewp, *vnewp; size_t attn_pos; - if (!w.shared) { - void* k = fresh((size_t)M*hd); if(!cactus_metal_encode_quant_matmul_m(k, xn, &w.k, M)) return bail(); - void* v = fresh((size_t)M*hd); if(!cactus_metal_encode_quant_matmul_m(v, xn, &w.v, M)) return bail(); - void* kn = fresh((size_t)M*hd); cactus_metal_encode_rms_norm(kn, k, w.k_norm, M, hd, RMS_EPS); - void* vn = fresh((size_t)M*hd); cactus_metal_encode_rms_norm(vn, v, w.global ? GP.vnorm_g : GP.vnorm, M, hd, RMS_EPS); - void* kr = fresh((size_t)M*hd); cactus_metal_encode_rope_m(kr, kn, rc, rs, 1, hd, M); - size_t clen = km[0]; - size_t vhd0 = GP.A[l].v_hdim > 0 ? GP.A[l].v_hdim : (size_t)hd; - size_t vmax0 = ((uint64_t*)w.vc)[1], ngV0 = (vhd0+31)/32; - char* kbase = (char*)w.kc; char* vbase = (char*)w.vc; - if (!cactus_metal_encode_kv_append_i8_m(kr, kbase+64, kbase+64+max_seq*kv_heads*hd, - (uint32_t)kv_heads, (uint32_t)hd, (uint32_t)clen, 32, M, - (size_t)M*hd*2, max_seq*kv_heads*hd, max_seq*kv_heads*ngK*sizeof(float))) - return bail(); - if (!cactus_metal_encode_kv_append_i8_m(vn, vbase+64, vbase+64+vmax0*kv_heads*vhd0, - (uint32_t)kv_heads, (uint32_t)vhd0, (uint32_t)clen, 32, M, - (size_t)M*vhd0*2, vmax0*kv_heads*vhd0, vmax0*kv_heads*ngV0*sizeof(float))) - return bail(); - km[0] = clen + M; ((uint64_t*)w.vc)[0] = clen + M; - hist = clen; newlen = M; knewp = kr; vnewp = vn; attn_pos = clen; - } else { - size_t clen = km[0]; - hist = clen; newlen = 0; knewp = nullptr; vnewp = nullptr; - attn_pos = (clen >= M) ? clen - M : 0; - } - - size_t v_hdim = AP.v_hdim > 0 ? AP.v_hdim : (size_t)hd; - size_t v_max = ((uint64_t*)w.vc)[1]; - size_t ngV = (v_hdim+31)/32; - size_t win = AP.window; - uint32_t ringv = (win > 0 && max_seq > 2*sink + 1) ? (uint32_t)(max_seq - 2*sink - 1) : 0u; - char* bk = (char*)w.kc; char* bv = (char*)w.vc; - void* attn = fresh((size_t)M*QD); - if (!cactus_metal_encode_attention_i8_prefill(attn, qr, knewp, vnewp, - bk+64, bv+64, bk+64+max_seq*kv_heads*hd, bv+64+v_max*kv_heads*v_hdim, - (uint32_t)NQH, (uint32_t)kv_heads, (uint32_t)hd, (uint32_t)v_hdim, - (uint32_t)hist, (uint32_t)newlen, (uint32_t)attn_pos, - (uint32_t)win, AP.causal?1u:0u, (uint32_t)M, ascale, - max_seq*kv_heads*hd, v_max*kv_heads*v_hdim, - max_seq*kv_heads*ngK*sizeof(float), v_max*kv_heads*ngV*sizeof(float), - (uint32_t)sink, ringv)) - return bail(); - - void* o = fresh((size_t)M*HID); if(!cactus_metal_encode_quant_matmul_m(o, attn, &w.o, M)) return bail(); - void* h1 = fresh((size_t)M*HID); cactus_metal_encode_rms_norm_add(h1, o, w.post_attn, h, M, HID, RMS_EPS); h = h1; - - const int MLP = w.mlp; - void* xn2 = fresh((size_t)M*HID); cactus_metal_encode_rms_norm(xn2, h, w.pre_ffn, M, HID, RMS_EPS); - void* gate= fresh((size_t)M*MLP); if(!cactus_metal_encode_quant_matmul_m(gate, xn2, &w.gate, M)) return bail(); - void* up = fresh((size_t)M*MLP); if(!cactus_metal_encode_quant_matmul_m(up, xn2, &w.up, M)) return bail(); - void* g3 = fresh((size_t)M*MLP); cactus_metal_encode_swiglu(g3, gate, up, (size_t)M*MLP, GATE_SCALE); - void* mo = fresh((size_t)M*HID); if(!cactus_metal_encode_quant_matmul_m(mo, g3, &w.down, M)) return bail(); - void* h2 = fresh((size_t)M*HID); cactus_metal_encode_rms_norm_add(h2, mo, w.post_ffn, h, M, HID, RMS_EPS); h = h2; - - void* ps = fresh((size_t)M*PLE_DIM); if(!cactus_metal_encode_quant_matmul_m(ps, h, &w.ple_down, M)) return bail(); - void* pled= fresh((size_t)M*PLE_DIM); - { uint32_t osh[2]={M,(uint32_t)PLE_DIM}; uint32_t sst[2]={(uint32_t)(NL*PLE_DIM),1}; - if(!cactus_metal_encode_strided_copy(pled, pleBase, osh, sst, 2, M*PLE_DIM, (uint32_t)(l*PLE_DIM), - (size_t)M*NL*PLE_DIM*2, (size_t)M*PLE_DIM*2)) return bail(); } - void* pm = fresh((size_t)M*PLE_DIM); cactus_metal_encode_swiglu(pm, ps, pled, (size_t)M*PLE_DIM, 1.0f); - void* pu = fresh((size_t)M*HID); if(!cactus_metal_encode_quant_matmul_m(pu, pm, &w.ple_up, M)) return bail(); - void* h3 = fresh((size_t)M*HID); cactus_metal_encode_rms_norm_add(h3, pu, w.post_ple, h, M, HID, RMS_EPS); h = h3; - - float ls = (float)(*(const __fp16*)w.scalar); - void* h4 = fresh((size_t)M*HID); cactus_metal_encode_scalar(2, h4, h, (size_t)M*HID, ls); h = h4; - } - - const void* hlast = (const char*)h + (size_t)(M-1)*HID*2; - size_t V = B(n-1).total_size; - void* fn = fresh(HID); cactus_metal_encode_rms_norm(fn, hlast, GP.final_norm, 1, HID, RMS_EPS); - void* code = fresh(HID); - void* lg = fresh(V); - if (!cactus_metal_encode_quant_matmul_ortho(lg, fn, code, &GP.lm_head)) return bail(); - if (!cactus_metal_encode_softcap(lg, lg, V, SOFTCAP)) { - cactus_metal_encode_scalar(3, lg, lg, V, SOFTCAP); - cactus_metal_encode_unary(1, lg, lg, V); - cactus_metal_encode_scalar(2, lg, lg, V, SOFTCAP); - } - B(n-1).set_external(lg); - cactus_metal_set_active(false); - cactus_metal_session_end(); - return true; -} void cactus_graph_on_destroy(const void* graph) { if (G.init && G.owner == graph) G.init = false; - if (GP.init && GP.owner == graph) GP.init = false; cactus_metal_invalidate_host_wraps(); } diff --git a/cactus-graph/src/ops_nn.cpp b/cactus-graph/src/ops_nn.cpp index 3582c5f5a..e0ad4de91 100644 --- a/cactus-graph/src/ops_nn.cpp +++ b/cactus-graph/src/ops_nn.cpp @@ -55,14 +55,12 @@ void compute_matmul_node(GraphNode& node, const std::vector(); CactusQuantMatrix mat = rhs_buffer.to_cq_matrix(); - const bool is_orthogonal = (rhs_buffer.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) != 0; - if (is_orthogonal) { + if (rhs_buffer.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) cactus_quant_orthogonal_matmul(&mat, lhs, static_cast(M), output); - } else if (cactus_backend_metal()) { + else if (cactus_backend_metal()) cactus_metal_quant_matmul(&mat, lhs, static_cast(M), output); - } else { + else cactus_quant_matmul(&mat, lhs, static_cast(M), output); - } } else { if (lhs_buffer.precision != Precision::FP16) { throw std::runtime_error("FP16 matmul requires FP16 activations (got precision " + std::to_string(static_cast(lhs_buffer.precision)) + ")"); diff --git a/cactus-kernels/CMakeLists.txt b/cactus-kernels/CMakeLists.txt index 0e57afeaa..b7891f2e9 100644 --- a/cactus-kernels/CMakeLists.txt +++ b/cactus-kernels/CMakeLists.txt @@ -74,9 +74,7 @@ if(APPLE) -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/embed_msl.cmake DEPENDS ${_msl_in} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/embed_msl.cmake VERBATIM) - add_custom_target(cactus_msl_header DEPENDS ${_msl_out}) - target_sources(cactus_kernels PRIVATE src/metal_backend.mm) - add_dependencies(cactus_kernels cactus_msl_header) + target_sources(cactus_kernels PRIVATE src/metal_backend.mm ${_msl_out}) target_include_directories(cactus_kernels PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/generated) set_source_files_properties(src/metal_backend.mm PROPERTIES COMPILE_FLAGS "-fobjc-arc") target_link_libraries(cactus_kernels PUBLIC "-framework Metal" "-framework Foundation") diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 4550784c8..14e687c42 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -21,13 +21,6 @@ bool cactus_metal_available(); void cactus_metal_set_active(bool active); bool cactus_metal_active_mode(); -bool cactus_metal_concurrent(); -void cactus_metal_barrier(); - -void cactus_metal_session_set_concurrent(bool on); -void cactus_metal_group_begin(); -void cactus_metal_group_end(); - void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, uint32_t M, __fp16* C); @@ -47,8 +40,6 @@ bool cactus_metal_encode_unary(int op_type, void* out, const void* in, size_t n) bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, size_t n, float scale); bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const void* sin, uint32_t heads, uint32_t head_dim); -bool cactus_metal_encode_rope_m(void* out, const void* x, const void* cos_m, const void* sin_m, - uint32_t heads, uint32_t head_dim, uint32_t M); bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps); bool cactus_metal_encode_rms_norm_rope(void* out, const void* x, const void* weight, @@ -72,11 +63,7 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M); bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* const* Ws, int B, void* const* codes); bool cactus_metal_encode_gemv_precoded(void* out, const void* code, const CactusQuantMatrix* W); -bool cactus_metal_encode_gemv_mega(void* const* outs, const void* x, const void* normw, float eps, - const CactusQuantMatrix* const* Ws, int B, const void* osw); -bool cactus_metal_encode_gemv_mega_ex(void* const* outs, const void* x, const void* normw, float eps, - const CactusQuantMatrix* const* Ws, int B, const void* osw, - bool pswi, float pscale); +bool cactus_metal_encode_transform_gemv(void* out, const void* x, const CactusQuantMatrix* W, const void* osw); bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, const CactusQuantMatrix* const* Ws, int B); bool cactus_metal_encode_swiglu_transform(void* code, const void* gate, const void* up, diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 35b6f93e7..3e1b2fae3 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -207,57 +207,28 @@ kernel void cq4_gemv_mr( } } -struct MegaU { uint K, ng, nw, N0, N1, N2, flags; float eps; float pscale; }; -kernel void cq4_gemv_mega( +struct TGU { uint K, ng, oswi; }; +kernel void cq4_transform_gemv( device const half* x [[buffer(0)]], - device const half* normw [[buffer(1)]], - device const half* recip [[buffer(2)]], - device const char* lsign0 [[buffer(3)]], device const char* lsign1 [[buffer(4)]], device const char* lsign2 [[buffer(5)]], - device const char* rsign0 [[buffer(6)]], device const char* rsign1 [[buffer(7)]], device const char* rsign2 [[buffer(8)]], - device const uint* perm0 [[buffer(9)]], device const uint* perm1 [[buffer(10)]], device const uint* perm2 [[buffer(11)]], - device const half* cbook0 [[buffer(12)]], device const half* cbook1 [[buffer(13)]], device const half* cbook2 [[buffer(14)]], - device const half* norms0 [[buffer(15)]], device const half* norms1 [[buffer(16)]], device const half* norms2 [[buffer(17)]], - device const uchar* packed0 [[buffer(18)]], device const uchar* packed1[[buffer(19)]], device const uchar* packed2[[buffer(20)]], - device half* y0 [[buffer(21)]], device half* y1 [[buffer(22)]], device half* y2 [[buffer(23)]], - device const half* osw [[buffer(24)]], - constant MegaU& U [[buffer(25)]], + device const half* recip [[buffer(1)]], + device const char* lsign [[buffer(2)]], + device const char* rsign [[buffer(3)]], + device const uint* perm [[buffer(4)]], + device const half* cbook [[buffer(5)]], + device const half* norms [[buffer(6)]], + device const uchar* packed [[buffer(7)]], + device half* y [[buffer(8)]], + device const half* osw [[buffer(9)]], + constant TGU& U [[buffer(10)]], uint tg [[threadgroup_position_in_grid]], uint sg [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], uint t [[thread_position_in_threadgroup]], threadgroup half* coded [[threadgroup(0)]], - threadgroup float* zwave [[threadgroup(1)]], - threadgroup float* red [[threadgroup(2)]]) + threadgroup float* zwave [[threadgroup(1)]]) { const uint K = U.K, ng = U.ng; - const bool do_norm = (U.flags & 1u) != 0u; - const bool do_oswi = (U.flags & 2u) != 0u; - const bool do_pswi = (U.flags & 4u) != 0u; - - uint n0 = tg*16u; - uint b, nbase; - if (n0 < U.N0) { b = 0u; nbase = 0u; } - else if (n0 < U.N0 + U.N1) { b = 1u; nbase = U.N0; } - else { b = 2u; nbase = U.N0 + U.N1; } - device const char* lsign = (b==0u)?lsign0:((b==1u)?lsign1:lsign2); - device const char* rsign = (b==0u)?rsign0:((b==1u)?rsign1:rsign2); - device const uint* perm = (b==0u)?perm0 :((b==1u)?perm1 :perm2 ); - device const half* cbook = (b==0u)?cbook0:((b==1u)?cbook1:cbook2); - device const half* norms = (b==0u)?norms0:((b==1u)?norms1:norms2); - device const uchar* packed= (b==0u)?packed0:((b==1u)?packed1:packed2); - device half* y = (b==0u)?y0 :((b==1u)?y1 :y2 ); - - threadgroup float* cb = red; - - float inv = 1.0f; - if (do_norm) { - float partial = 0; - for (uint i=t; i0u; s>>=1){ if (t>4)&0xF] + (float)c0.z*cb[(w[0]>>8)&0xF] + (float)c0.w*cb[(w[0]>>12)&0xF] @@ -325,8 +283,8 @@ kernel void cq4_gemv_mega( for (uint r=0u;r<2u;r++){ float a = simd_sum(acc[r]); if (lane==0u){ - uint n = nloc0+r; - if (do_oswi) { + uint n = n0+r; + if (U.oswi != 0u) { float gg = gelu_tanh((float)(half)a) * (float)osw[n]; y[n] = (half)clamp(gg,-65504.0f,65504.0f); } else { @@ -987,17 +945,6 @@ kernel void rope_f16(device const half* x [[buffer(0)]], device half* out [[buff out[gid]=(half)((float)x[gid]*(float)cs[d] + rot*(float)sn[d]); } -kernel void rope_f16_m(device const half* x [[buffer(0)]], device half* out [[buffer(1)]], - device const half* cs [[buffer(2)]], device const half* sn [[buffer(3)]], - constant uint& heads [[buffer(4)]], constant uint& hd [[buffer(5)]], - constant uint& M [[buffer(6)]], uint gid [[thread_position_in_grid]]) { - uint per=heads*hd, total=M*per; if (gid>=total) return; - uint m=gid/per, r=gid%per, d=r%hd, hh=hd/2; - uint base=m*per + (r/hd)*hd; - float rot = (d #include #include -#include #include "cactus_kernels_msl.h" @@ -42,7 +41,7 @@ id psoAttnPre=nil, psoAttnPreMma2=nil, psoAttnPreHd256=nil; id psoKvAppendM=nil, psoKvAppendRingM=nil; id psoSlideS=nil, psoSlideR=nil, psoSlideRM=nil; - id psoRope=nil, psoRopeM=nil, psoRmsRope=nil; + id psoRope=nil, psoRmsRope=nil; id psoArgmax=nil; id psoArgmaxP=nil, psoArgmaxF=nil, psoSoftcap=nil, psoAdjust=nil; id psoGather=nil; @@ -50,17 +49,12 @@ bool ok = false; MetalCtx() { @autoreleasepool { - const bool prof = getenv("CACTUS_PROF_INIT")!=nullptr; - auto t0 = std::chrono::high_resolution_clock::now(); - auto ms = [&](){ auto n=std::chrono::high_resolution_clock::now(); return std::chrono::duration(n-t0).count(); }; dev = MTLCreateSystemDefaultDevice(); if (!dev) return; queue = [dev newCommandQueue]; NSError* err = nil; id lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kCactusMSL] options:nil error:&err]; - if (prof) fprintf(stderr,"[prof-init] MSL compile: %.1fms\n", ms()); - if (!lib) { if (err) fprintf(stderr,"[cactus-metal] MSL compile failed: %s\n",[[err localizedDescription] UTF8String]); return; } auto pso = [&](const char* name) -> id { NSError* e=nil; @@ -70,7 +64,7 @@ return p; }; psoT=pso("cq4_transform"); psoTsimd=pso("cq4_transform_simd"); psoG=pso("cq4_gemv"); psoTbatch=pso("cq4_transform_batch"); psoGmr=pso("cq4_gemv_mr"); - psoMega=pso("cq4_gemv_mega"); psoSwiT=pso("cq4_swiglu_transform"); psoCat=pso("cq4_gemv_cat"); + psoMega=pso("cq4_transform_gemv"); psoSwiT=pso("cq4_swiglu_transform"); psoCat=pso("cq4_gemv_cat"); psoTm=pso("cq4_transform_m"); psoGm=pso("cq4_gemm"); psoGmma=pso("cq4_gemm_mma"); psoGdense=pso("cq4_gemm_dense_f16"); psoDeq=pso("cq4_dequant_f16"); psoGmrLow=pso("cq_gemv_mr_lowbit"); psoDeqLow=pso("cq_dequant_lowbit_f16"); @@ -88,14 +82,13 @@ psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoAttnPreHd256=pso("attn_prefill_mma_hd256"); psoKvAppendM=pso("kv_append_i8_m"); psoKvAppendRingM=pso("kv_append_ring_i8_m"); psoSlideS=pso("kv_slide_save"); psoSlideR=pso("kv_slide_restore"); psoSlideRM=pso("kv_slide_restore_m"); - psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); psoRopeM=pso("rope_f16_m"); psoRmsRope=pso("rms_norm_rope_f16"); + psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); psoRmsRope=pso("rms_norm_rope_f16"); psoArgmax=pso("argmax_logits"); psoArgmaxP=pso("argmax_part"); psoArgmaxF=pso("argmax_final"); psoSoftcap=pso("softcap_f16"); psoAdjust=pso("adjust_logits"); - if (prof) fprintf(stderr,"[prof-init] pso creation: %.1fms total\n", ms()); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 - &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoRopeM&&psoArgmax&&psoGather; + &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; }} }; @@ -224,68 +217,17 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, inline size_t bucket(size_t b) { return (b + 4095) & ~size_t(4095); } -struct GemmProf { - double gpu_ns=0; long cmdbufs=0; - ~GemmProf(){ if(std::getenv("CACTUS_PROF_GPU") && cmdbufs>0) - fprintf(stderr,"[PROF gpu-busy] cmdbufs=%ld total_gpu=%.2fms\n", cmdbufs, gpu_ns/1e6); } -}; -static GemmProf g_gemmprof; -static const bool g_prof_gpu = (std::getenv("CACTUS_PROF_GPU") != nullptr); - -static const bool g_prof_ops = (std::getenv("CACTUS_PROF_OPS") != nullptr); -static const bool g_gemv_mr = (std::getenv("CACTUS_NO_GEMV_MR") == nullptr); static const uint32_t g_mr_rows = 8u * 2u; - -static const bool g_skip_mm=(std::getenv("CACTUS_SKIP_MM")!=nullptr), g_skip_attn=(std::getenv("CACTUS_SKIP_ATTN")!=nullptr), g_skip_norm=(std::getenv("CACTUS_SKIP_NORM")!=nullptr); -static const bool g_skip_transform=(std::getenv("CACTUS_SKIP_TRANSFORM")!=nullptr); static const bool g_predequant=(std::getenv("CACTUS_NO_PREDEQUANT")==nullptr); -static const char* g_op = "other"; -struct OpProf { - std::map> t; - ~OpProf(){ if(!g_prof_ops || t.empty()) return; - double tot=0; for(auto&kv:t) tot+=kv.second.first; - fprintf(stderr,"[PROF ops] total=%.1fms\n", tot/1e6); - for(auto&kv:t) fprintf(stderr," %-12s %8.2fms %5.1f%% (n=%ld)\n", - kv.first.c_str(), kv.second.first/1e6, 100.0*kv.second.first/tot, kv.second.second); } -}; -static OpProf g_opprof; - -static const bool g_concurrent = (std::getenv("CACTUS_GPU_CONCURRENT") != nullptr); -static const bool g_no_groups = (std::getenv("CACTUS_NO_GROUPS") != nullptr); -bool g_session_concurrent = false; -bool g_group = false; id ensureEncoder() { if (!g_enc) { g_cmd = [ctx().queue commandBuffer]; - g_enc = (g_concurrent || g_session_concurrent) - ? [g_cmd computeCommandEncoderWithDispatchType:MTLDispatchTypeConcurrent] - : [g_cmd computeCommandEncoder]; + g_enc = [g_cmd computeCommandEncoder]; } return g_enc; } -inline void hard_barrier() { - if ((g_concurrent || g_session_concurrent) && g_enc) - [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; -} - -inline void barrier() { - static const bool force = (std::getenv("CACTUS_FORCE_BARRIER") != nullptr); - if (g_prof_ops && g_enc) { - [g_enc endEncoding]; [g_cmd commit]; [g_cmd waitUntilCompleted]; - g_opprof.t[g_op].first += (g_cmd.GPUEndTime - g_cmd.GPUStartTime)*1e9; - g_opprof.t[g_op].second += 1; - g_enc = nil; g_cmd = nil; - for (id b : g_pending) g_free[(size_t)b.length].push_back(b); - g_pending.clear(); - g_op = "other"; - return; - } - if (g_group) return; - if (g_concurrent || g_session_concurrent || force) [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; -} - id recycled(size_t bytes) { size_t bk = bucket(bytes ? bytes : 1); auto it = g_free.find(bk); @@ -346,24 +288,10 @@ inline void setBufAt(const void* p, size_t bytes, int idx) { void cactus_metal_set_active(bool a) { g_active = a; } bool cactus_metal_active_mode() { return ctx().ok && g_active; } -bool cactus_metal_concurrent() { return g_concurrent; } -void cactus_metal_barrier() { if ((g_concurrent || g_session_concurrent) && g_enc) [g_enc memoryBarrierWithScope:MTLBarrierScopeBuffers]; } - -void cactus_metal_session_set_concurrent(bool on) { - if (g_no_groups) return; - if (g_enc) return; - g_session_concurrent = on; - if (!on) g_group = false; -} -void cactus_metal_group_begin() { if (g_session_concurrent && !g_prof_ops) g_group = true; } -void cactus_metal_group_end() { if (g_group) { g_group = false; barrier(); } } - void cactus_metal_session_begin() {} void cactus_metal_session_sync() { if (g_enc) { [g_enc endEncoding]; - if (g_prof_gpu) [g_cmd addCompletedHandler:^(id cb){ - g_gemmprof.gpu_ns += (cb.GPUEndTime-cb.GPUStartTime)*1e9; g_gemmprof.cmdbufs += 1; }]; [g_cmd commit]; [g_cmd waitUntilCompleted]; g_enc = nil; g_cmd = nil; } @@ -384,8 +312,6 @@ void cactus_metal_invalidate_host_wraps() { void cactus_metal_session_flush() { if (!g_enc) return; [g_enc endEncoding]; - if (g_prof_gpu) [g_cmd addCompletedHandler:^(id cb){ - g_gemmprof.gpu_ns += (cb.GPUEndTime-cb.GPUStartTime)*1e9; g_gemmprof.cmdbufs += 1; }]; [g_cmd commit]; g_enc = nil; g_cmd = nil; } @@ -416,7 +342,6 @@ bool cactus_metal_encode_copy(void* out, const void* in, size_t bytes) { setBufAt(out, bytes, 1); [g_enc setBytes:&n length:4 atIndex:2]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } bool cactus_metal_encode_binary(int op, void* out, const void* a, const void* b, size_t n) { @@ -427,7 +352,6 @@ bool cactus_metal_encode_binary(int op, void* out, const void* a, const void* b, setBufAt(a, n*2, 0); setBufAt(b, n*2, 1); setBufAt(out, n*2, 2); [g_enc setBytes:&nn length:4 atIndex:3]; [g_enc setBytes:&o length:4 atIndex:4]; [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } bool cactus_metal_encode_scalar(int op, void* out, const void* in, size_t n, float param) { @@ -438,7 +362,6 @@ bool cactus_metal_encode_scalar(int op, void* out, const void* in, size_t n, flo setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; [g_enc setBytes:&p length:4 atIndex:4]; [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } bool cactus_metal_encode_unary(int op, void* out, const void* in, size_t n) { @@ -449,47 +372,28 @@ bool cactus_metal_encode_unary(int op, void* out, const void* in, size_t n) { setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, size_t n, float scale) { if (!ctx().ok) return false; ensureEncoder(); - g_op="swiglu"; uint32_t nn=(uint32_t)n; float s=scale; [g_enc setComputePipelineState:ctx().psoSwiglu]; setBufAt(gate, n*2, 0); setBufAt(up, n*2, 1); setBufAt(out, n*2, 2); [g_enc setBytes:&nn length:4 atIndex:3]; [g_enc setBytes:&s length:4 atIndex:4]; [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const void* sin, uint32_t heads, uint32_t head_dim) { if (!ctx().ok) return false; ensureEncoder(); - g_op="rope"; uint32_t total=heads*head_dim; [g_enc setComputePipelineState:ctx().psoRope]; setBufAt(x, (size_t)total*2, 0); setBufAt(out, (size_t)total*2, 1); setBufAt(cos, (size_t)head_dim*2, 2); setBufAt(sin, (size_t)head_dim*2, 3); [g_enc setBytes:&heads length:4 atIndex:4]; [g_enc setBytes:&head_dim length:4 atIndex:5]; [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); - return true; -} -bool cactus_metal_encode_rope_m(void* out, const void* x, const void* cos_m, const void* sin_m, - uint32_t heads, uint32_t head_dim, uint32_t M) { - if (!ctx().ok) return false; - ensureEncoder(); - g_op="rope"; - uint32_t total=M*heads*head_dim; - [g_enc setComputePipelineState:ctx().psoRopeM]; - setBufAt(x, (size_t)total*2, 0); setBufAt(out, (size_t)total*2, 1); - setBufAt(cos_m, (size_t)M*head_dim*2, 2); setBufAt(sin_m, (size_t)M*head_dim*2, 3); - [g_enc setBytes:&heads length:4 atIndex:4]; [g_enc setBytes:&head_dim length:4 atIndex:5]; [g_enc setBytes:&M length:4 atIndex:6]; - [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } bool cactus_metal_encode_rms_norm_rope(void* out, const void* x, const void* weight, @@ -497,7 +401,6 @@ bool cactus_metal_encode_rms_norm_rope(void* out, const void* x, const void* wei uint32_t heads, uint32_t head_dim, float eps) { if (!ctx().ok || !ctx().psoRmsRope) return false; ensureEncoder(); - g_op="rmsnorm"; uint32_t total=heads*head_dim; uint32_t hd=head_dim; float e=eps; [g_enc setComputePipelineState:ctx().psoRmsRope]; setBufAt(x, (size_t)total*2, 0); setBufAt(weight, (size_t)head_dim*2, 1); @@ -505,50 +408,43 @@ bool cactus_metal_encode_rms_norm_rope(void* out, const void* x, const void* wei setBufAt(out, (size_t)total*2, 4); [g_enc setBytes:&hd length:4 atIndex:5]; [g_enc setBytes:&e length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; - if(!g_skip_norm) [g_enc dispatchThreadgroups:MTLSizeMake(heads,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(heads,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps) { if (!ctx().ok) return false; ensureEncoder(); - g_op="rmsnorm"; uint32_t d=(uint32_t)dim; float e=eps; [g_enc setComputePipelineState:ctx().psoRms]; setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(out, rows*dim*2, 2); [g_enc setBytes:&d length:4 atIndex:3]; [g_enc setBytes:&e length:4 atIndex:4]; [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; - if(!g_skip_norm) [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* weight, const void* res, size_t rows, size_t dim, float eps) { if (!ctx().ok) return false; ensureEncoder(); - g_op="rmsnorm"; uint32_t d=(uint32_t)dim; float e=eps; [g_enc setComputePipelineState:ctx().psoRmsAdd]; setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(res, rows*dim*2, 2); setBufAt(out, rows*dim*2, 3); [g_enc setBytes:&d length:4 atIndex:4]; [g_enc setBytes:&e length:4 atIndex:5]; [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; - if(!g_skip_norm) [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_rms_norm_add_scale(void* out, const void* in, const void* weight, const void* res, size_t rows, size_t dim, float eps, float out_scale) { if (!ctx().ok || !ctx().psoRmsAddScale) return false; ensureEncoder(); - g_op="rmsnorm"; uint32_t d=(uint32_t)dim; float e=eps, os=out_scale; [g_enc setComputePipelineState:ctx().psoRmsAddScale]; setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(res, rows*dim*2, 2); setBufAt(out, rows*dim*2, 3); [g_enc setBytes:&d length:4 atIndex:4]; [g_enc setBytes:&e length:4 atIndex:5]; [g_enc setBytes:&os length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; - if(!g_skip_norm) [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_rms_norm_add_rms(void* h_out, void* xn_out, const void* in, const void* w1, @@ -556,22 +452,19 @@ bool cactus_metal_encode_rms_norm_add_rms(void* h_out, void* xn_out, const void* size_t rows, size_t dim, float eps, float out_scale) { if (!ctx().ok || !ctx().psoRmsAddRms) return false; ensureEncoder(); - g_op="rmsnorm"; uint32_t d=(uint32_t)dim; float e=eps, os=out_scale; [g_enc setComputePipelineState:ctx().psoRmsAddRms]; setBufAt(in, rows*dim*2, 0); setBufAt(w1, dim*2, 1); setBufAt(res, rows*dim*2, 2); setBufAt(h_out, rows*dim*2, 3); setBufAt(w2, dim*2, 4); setBufAt(xn_out, rows*dim*2, 5); [g_enc setBytes:&d length:4 atIndex:6]; [g_enc setBytes:&e length:4 atIndex:7]; [g_enc setBytes:&os length:4 atIndex:8]; [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; - if(!g_skip_norm) [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) { if (!ctx().ok) return false; ensureEncoder(); - g_op="argmax"; uint32_t V = vocab; if (ctx().psoArgmaxP && ctx().psoArgmaxF && V >= 32768u) { const uint32_t NP = 128u, T = 256u; @@ -585,7 +478,6 @@ bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) [g_enc setThreadgroupMemoryLength:T*sizeof(uint) atIndex:1]; [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:2]; [g_enc dispatchThreadgroups:MTLSizeMake(NP,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; - hard_barrier(); [g_enc setComputePipelineState:ctx().psoArgmaxF]; [g_enc setBuffer:part offset:0 atIndex:0]; setBufAt(out3, 3*sizeof(float), 1); @@ -594,7 +486,6 @@ bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) [g_enc setThreadgroupMemoryLength:NP*sizeof(uint) atIndex:1]; [g_enc setThreadgroupMemoryLength:NP*sizeof(float) atIndex:2]; [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(NP,1,1)]; - barrier(); return true; } uint32_t T = 1024; @@ -606,7 +497,6 @@ bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) [g_enc setThreadgroupMemoryLength:T*sizeof(uint) atIndex:1]; [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:2]; [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; - barrier(); return true; } bool cactus_metal_encode_adjust_logits(void* logits, size_t vocab, @@ -616,7 +506,6 @@ bool cactus_metal_encode_adjust_logits(void* logits, size_t vocab, if (!ctx().ok || !ctx().psoAdjust) return false; if (n_recent > 4096u || n_bias > 65536u) return false; ensureEncoder(); - g_op="other"; id rb = ctx().dummy, bi = ctx().dummy, bv = ctx().dummy; if (n_recent) { rb = recycled((size_t)n_recent*4); std::memcpy([rb contents], recent, (size_t)n_recent*4); } if (n_bias) { bi = recycled((size_t)n_bias*4); std::memcpy([bi contents], bias_ids, (size_t)n_bias*4); @@ -630,7 +519,6 @@ bool cactus_metal_encode_adjust_logits(void* logits, size_t vocab, [g_enc setBuffer:bi offset:0 atIndex:2]; [g_enc setBuffer:bv offset:0 atIndex:3]; [g_enc setBytes:&U length:sizeof(U) atIndex:4]; [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(1024,1,1)]; - barrier(); return true; } @@ -642,7 +530,6 @@ bool cactus_metal_encode_softcap(void* out, const void* in, size_t n, float cap) setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&c length:4 atIndex:3]; [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_prec, size_t n) { @@ -662,7 +549,6 @@ bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_pr setBufAt(in, n*esz(in_prec), 0); setBufAt(out, n*esz(out_prec), 1); [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } static inline bool quant_fast_eligible(const CactusQuantMatrix* W) { @@ -679,7 +565,7 @@ static inline bool quant_lowbit_eligible(const CactusQuantMatrix* W) { } bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W) { - const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; + const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; if (ctx().ok && (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->rotation && ng==1 && gs==K && (N%4)==0) { static void* ortho_code = nullptr; static uint32_t ortho_code_k = 0; if (ortho_code_k < K) { ortho_code = cactus_metal_alloc_shared((size_t)K*sizeof(__fp16)); ortho_code_k = K; } @@ -695,7 +581,6 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu if (!g_code_buf || (size_t)g_code_buf.length < code_bytes) g_code_buf = [ctx().dev newBufferWithLength:code_bytes options:MTLResourceStorageModeShared]; ensureEncoder(); - g_op="matmul"; bool simdT = (gs==128u && ctx().psoTsimd); [g_enc setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; setBufAt(lhs, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; @@ -703,7 +588,6 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu [g_enc setBuffer:rw.perm offset:0 atIndex:4]; [g_enc setBuffer:g_code_buf offset:0 atIndex:5]; [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; - hard_barrier(); if (lowbit && bits == 2u && gs == 128u && ctx().psoActQ && ctx().psoG2i8 && rw.cb_i8) { id actq = recycled((size_t)ng*gs); id ascl = recycled((size_t)ng*sizeof(float)); @@ -712,7 +596,6 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu [g_enc setBuffer:actq offset:0 atIndex:1]; [g_enc setBuffer:ascl offset:0 atIndex:2]; [g_enc setBytes:&gs length:4 atIndex:3]; [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; - hard_barrier(); [g_enc setComputePipelineState:ctx().psoG2i8]; [g_enc setBuffer:actq offset:0 atIndex:0]; [g_enc setBuffer:ascl offset:0 atIndex:1]; [g_enc setBuffer:rw.packed offset:0 atIndex:2]; [g_enc setBuffer:rw.cb_i8 offset:0 atIndex:3]; @@ -720,11 +603,10 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu setBufAt(out, (size_t)N*2, 5); [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&N length:4 atIndex:7]; [g_enc setBytes:&rw.cb_scale length:4 atIndex:8]; - if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake((N+7u)/8u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake((N+7u)/8u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } - bool umr = g_gemv_mr && ctx().psoGmr && (N % g_mr_rows == 0u); + bool umr = ctx().psoGmr && (N % g_mr_rows == 0u); [g_enc setComputePipelineState:(lowbit?ctx().psoGmrLow:(umr?ctx().psoGmr:ctx().psoG))]; [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:0 atIndex:1]; [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; @@ -733,10 +615,8 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu [g_enc setBytes:&pgbw length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; if (lowbit) [g_enc setBytes:&bits length:4 atIndex:9]; uint32_t ROWS=8; - g_op="matmul"; uint32_t grid = (lowbit || umr) ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; - if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; return true; } @@ -748,7 +628,6 @@ bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* ResW* rw[3] = { &r0, &r0, &r0 }; for (int b=1;b 3) return false; - const uint32_t gs=Ws[0]->group_size, K=Ws[0]->K, ng=Ws[0]->num_groups; - if (gs != 128u) return false; - uint32_t Ns[3]={0,0,0}; - for (int i=0;iK != K || (Ws[i]->N % 16u) != 0u) return false; - Ns[i]=Ws[i]->N; - } - ResW* rw[3]; rw[0]=&resident(Ws[0]); rw[1]=rw[0]; rw[2]=rw[0]; - for (int i=1;igroup_size, K=W->K, ng=W->num_groups, N=W->N; + if (gs != 128u || !quant_fast_eligible(W) || (N % 16u) != 0u) return false; + ResW& rw = resident(W); ensureEncoder(); - g_op="matmul"; - struct { uint32_t K, ng, nw, N0, N1, N2, flags; float eps, pscale; } U = - { K, ng, (uint32_t)B, Ns[0], Ns[1], Ns[2], - (uint32_t)((normw?1u:0u) | ((osw && !pswi)?2u:0u) | (pswi?4u:0u)), eps, pscale }; + struct { uint32_t K, ng, oswi; } U = { K, ng, (uint32_t)(osw ? 1 : 0) }; [g_enc setComputePipelineState:ctx().psoMega]; setBufAt(x, (size_t)K*2, 0); - if (normw) setBufAt(normw, (size_t)K*2, 1); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; - [g_enc setBuffer:rw[0]->recip offset:0 atIndex:2]; - for (int i=0;i<3;i++){ - ResW* R=rw[i]; - [g_enc setBuffer:R->lsign offset:0 atIndex:3+i]; - [g_enc setBuffer:R->rsign offset:0 atIndex:6+i]; - [g_enc setBuffer:R->perm offset:0 atIndex:9+i]; - [g_enc setBuffer:R->codebook offset:0 atIndex:12+i]; - [g_enc setBuffer:R->norms offset:0 atIndex:15+i]; - [g_enc setBuffer:R->packed offset:0 atIndex:18+i]; - setBufAt(outs[igroup_size != 128u || !quant_fast_eligible(W)) return false; const uint32_t K=W->K, ng=W->num_groups; ResW& rw = resident(W); ensureEncoder(); - g_op="transform"; float s = scale; [g_enc setComputePipelineState:ctx().psoSwiT]; setBufAt(gate, (size_t)K*2, 0); setBufAt(up, (size_t)K*2, 1); @@ -852,7 +701,6 @@ bool cactus_metal_encode_swiglu_transform(void* code, const void* gate, const vo [g_enc setBytes:&s length:4 atIndex:7]; [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; - barrier(); return true; } @@ -869,7 +717,6 @@ bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, ResW* rw[3]; rw[0]=&resident(Ws[0]); rw[1]=rw[0]; rw[2]=rw[0]; for (int i=1;i1024u?1024u:gs,1,1)]; - hard_barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; if ((g_predequant && ctx().psoGdense && ctx().psoDeq) || lowbit) { if (!rw.wf16) { @@ -929,15 +772,12 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus [g_enc setBytes:&pgb length:4 atIndex:6]; [g_enc setBytes:&N length:4 atIndex:7]; if (lowbit) [g_enc setBytes:&bits length:4 atIndex:8]; [g_enc dispatchThreads:MTLSizeMake((size_t)N*ng,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - hard_barrier(); } [g_enc setComputePipelineState:ctx().psoGdense]; [g_enc setBuffer:g_code_buf_m offset:0 atIndex:0]; [g_enc setBuffer:rw.wf16 offset:0 atIndex:1]; setBufAt(out, (size_t)M*N*2, 2); [g_enc setBytes:&K length:4 atIndex:3]; [g_enc setBytes:&N length:4 atIndex:4]; [g_enc setBytes:&M length:4 atIndex:5]; - g_op="matmul"; - if (!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; return true; } @@ -948,14 +788,12 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus setBufAt(out, (size_t)M*N*2, 4); [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; [g_enc setBytes:&M length:4 atIndex:9]; - g_op="matmul"; - if (!g_skip_mm) { if (mma) { + if (mma) { [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; } else { uint32_t ROWS=8, MT=16; [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,(M+MT-1)/MT,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; - } } - barrier(); + } return true; } @@ -986,9 +824,8 @@ bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* co [g_enc setBytes:&K length:4 atIndex:4]; [g_enc dispatchThreads:MTLSizeMake(K,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; } - hard_barrier(); - bool umr = g_gemv_mr && ctx().psoGmr && (N % g_mr_rows == 0u); + bool umr = ctx().psoGmr && (N % g_mr_rows == 0u); [g_enc setComputePipelineState:(lowbit?ctx().psoGmrLow:(umr?ctx().psoGmr:ctx().psoG))]; setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:0 atIndex:1]; [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; @@ -998,8 +835,7 @@ bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* co if (lowbit) [g_enc setBytes:&bits length:4 atIndex:9]; uint32_t ROWS=8; uint32_t grid = (lowbit || umr) ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; - if(!g_skip_mm) [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; return true; } @@ -1021,7 +857,6 @@ bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQu [g_enc setBytes:&scale length:4 atIndex:8]; [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc setThreadgroupMemoryLength:(size_t)K*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(K/8u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } [g_enc setComputePipelineState:ctx().psoEmbO]; @@ -1032,7 +867,6 @@ bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQu uint32_t T = K>512u ? 512u : K; [g_enc setThreadgroupMemoryLength:(size_t)K*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; - barrier(); return true; } @@ -1072,7 +906,6 @@ bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const Cactu [g_enc setBytes:&gs length:4 atIndex:8]; [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc setThreadgroupMemoryLength:(size_t)gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(gs,1,1)]; - barrier(); return true; } @@ -1104,7 +937,6 @@ bool cactus_metal_encode_embedding_ortho_m(void* out, const CactusQuantMatrix* W setBufAt(out, (size_t)M*K*2, 6); [g_enc setBytes:&K length:4 atIndex:7]; [g_enc setBytes:&M length:4 atIndex:8]; [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc dispatchThreadgroups:MTLSizeMake(((K+15)/16)*((M+15)/16),1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } @@ -1133,7 +965,6 @@ bool cactus_metal_encode_embedding_hadamard_m(void* out, const CactusQuantMatrix [g_enc setBytes:&bits length:4 atIndex:11]; [g_enc setThreadgroupMemoryLength:(size_t)gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs,1,1)]; - barrier(); return true; } @@ -1153,7 +984,6 @@ bool cactus_metal_encode_gather_f16(void* out, const void* table, size_t table_b setBufAt(out, (size_t)n*2, 2); [g_enc setBytes:&D length:4 atIndex:3]; [g_enc setBytes:&n length:4 atIndex:4]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } @@ -1166,7 +996,6 @@ bool cactus_metal_encode_attention_i8( if (!ctx().ok) return false; if (kv_end <= kv_start || (num_q_heads % num_kv_heads) != 0) return false; ensureEncoder(); - g_op="attention"; auto setCache = [&](const void* p, size_t bytes, int idx) { if (p && bytes) setBufAt(p, bytes, idx); @@ -1202,15 +1031,13 @@ bool cactus_metal_encode_attention_i8( [g_enc setBuffer:partO offset:0 atIndex:16]; [g_enc setBuffer:partML offset:0 atIndex:17]; [g_enc setBytes:&nwg length:4 atIndex:18]; [g_enc setThreadgroupMemoryLength:((size_t)nsg*v_hdim + 2*nsg)*sizeof(float) atIndex:0]; - if(!g_skip_attn)[g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; - hard_barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; if (nwg > 1u) { [g_enc setComputePipelineState:ctx().psoAttnC]; [g_enc setBuffer:partO offset:0 atIndex:0]; [g_enc setBuffer:partML offset:0 atIndex:1]; setBufAt(out, (size_t)num_q_heads*v_hdim*2, 2); [g_enc setBytes:&v_hdim length:4 atIndex:3]; [g_enc setBytes:&nwg length:4 atIndex:4]; [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); } return true; } @@ -1226,7 +1053,6 @@ bool cactus_metal_encode_attention_fused_i8( if (!ctx().ok || !ctx().psoAttnF) return false; if (kv_end <= kv_start) return false; ensureEncoder(); - g_op="attention"; const uint32_t T = 256, nsg = T / 32u; const uint32_t R = kv_end - kv_start; uint32_t nwg = R / 24u; if (nwg < 1u) nwg = 1u; if (nwg > 32u) nwg = 32u; @@ -1252,8 +1078,7 @@ bool cactus_metal_encode_attention_fused_i8( [g_enc setBytes:&U length:sizeof(U) atIndex:15]; [g_enc setThreadgroupMemoryLength:((size_t)nsg*vhd + 2*nsg + 256)*sizeof(float) atIndex:0]; [g_enc setThreadgroupMemoryLength:(size_t)(2*hd + vhd)*2 atIndex:1]; - if(!g_skip_attn)[g_enc dispatchThreadgroups:MTLSizeMake(nqh*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; - hard_barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(nqh*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; if (nwg > 1u) { [g_enc setComputePipelineState:ctx().psoAttnC]; [g_enc setBuffer:partO offset:0 atIndex:0]; [g_enc setBuffer:partML offset:0 atIndex:1]; @@ -1261,7 +1086,6 @@ bool cactus_metal_encode_attention_fused_i8( [g_enc setBytes:&vhd length:4 atIndex:3]; [g_enc setBytes:&nwg length:4 atIndex:4]; [g_enc dispatchThreadgroups:MTLSizeMake(nqh,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; } - barrier(); return true; } @@ -1277,15 +1101,13 @@ bool cactus_metal_encode_attention_i8_prefill( const uint32_t T = 128; if (total_keys == 0 || M == 0 || (num_q_heads % num_kv_heads) != 0) return false; uint32_t maxsc = (ring > 0u) ? ((total_keys > sink + ring) ? (sink + ring) : total_keys) : 256u; - static const bool en_hd256 = (std::getenv("CACTUS_NO_HD256") == nullptr); bool mma2 = (ring == 0u && is_causal && total_keys >= 2048u && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u); if (!mma2 && is_causal && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u && window == 0u) mma2 = true; - bool hd256 = (en_hd256 && ctx().psoAttnPreHd256 != nil && is_causal && head_dim == 256u && v_hdim == 256u + bool hd256 = (ctx().psoAttnPreHd256 != nil && is_causal && head_dim == 256u && v_hdim == 256u && num_q_heads == 8u && num_kv_heads == 1u); if (mma2 || hd256) { ensureEncoder(); - g_op="attention"; [g_enc setComputePipelineState:(hd256 ? ctx().psoAttnPreHd256 : ctx().psoAttnPreMma2)]; setBufAt(q, (size_t)M*num_q_heads*head_dim*2, 0); if (new_len > 0 && knew && vnew) { @@ -1306,8 +1128,7 @@ bool cactus_metal_encode_attention_i8_prefill( [g_enc setBytes:&M length:4 atIndex:16]; if (hd256) { [g_enc setBytes:&sink length:4 atIndex:17]; [g_enc setBytes:&ring length:4 atIndex:18]; } uint32_t mtiles = (M + 7u)/8u; - if(!g_skip_attn)[g_enc dispatchThreadgroups:MTLSizeMake(mtiles, 1, 1) threadsPerThreadgroup:MTLSizeMake(hd256?256u:512u,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(mtiles, 1, 1) threadsPerThreadgroup:MTLSizeMake(hd256?256u:512u,1,1)]; return true; } ensureEncoder(); @@ -1332,8 +1153,7 @@ bool cactus_metal_encode_attention_i8_prefill( [g_enc setBytes:&maxsc length:4 atIndex:18]; [g_enc setBytes:&sink length:4 atIndex:19]; [g_enc setBytes:&ring length:4 atIndex:20]; [g_enc setThreadgroupMemoryLength:((size_t)maxsc + T)*sizeof(float) atIndex:0]; - if(!g_skip_attn)[g_enc dispatchThreadgroups:MTLSizeMake(M*num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; - barrier(); + [g_enc dispatchThreadgroups:MTLSizeMake(M*num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; return true; } @@ -1346,7 +1166,6 @@ bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* [g_enc setBytes:oshape length:ndim*4 atIndex:2]; [g_enc setBytes:sstride length:ndim*4 atIndex:3]; [g_enc setBytes:&ndim length:4 atIndex:4]; [g_enc setBytes:&total length:4 atIndex:5]; [g_enc setBytes:&base length:4 atIndex:6]; [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } bool cactus_metal_encode_bcast_binary(int op, void* out, const void* a, const void* b, @@ -1360,7 +1179,6 @@ bool cactus_metal_encode_bcast_binary(int op, void* out, const void* a, const vo [g_enc setBytes:oshape length:ndim*4 atIndex:3]; [g_enc setBytes:astride length:ndim*4 atIndex:4]; [g_enc setBytes:bstride length:ndim*4 atIndex:5]; [g_enc setBytes:&ndim length:4 atIndex:6]; [g_enc setBytes:&total length:4 atIndex:7]; [g_enc setBytes:&o length:4 atIndex:8]; [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } bool cactus_metal_encode_strided_scatter(void* out, const void* in, const uint32_t* ishape, @@ -1372,7 +1190,6 @@ bool cactus_metal_encode_strided_scatter(void* out, const void* in, const uint32 [g_enc setBytes:ishape length:ndim*4 atIndex:2]; [g_enc setBytes:ostride length:ndim*4 atIndex:3]; [g_enc setBytes:&ndim length:4 atIndex:4]; [g_enc setBytes:&total length:4 atIndex:5]; [g_enc setBytes:&base length:4 atIndex:6]; [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - barrier(); return true; } bool cactus_metal_encode_kv_append_i8(const void* src, void* int8base, void* scalebase, @@ -1381,13 +1198,11 @@ bool cactus_metal_encode_kv_append_i8(const void* src, void* int8base, void* sca if (!ctx().ok) return false; uint32_t num_groups = (hdim + group_size - 1)/group_size, n = kv_heads*num_groups; ensureEncoder(); - g_op="kv"; [g_enc setComputePipelineState:ctx().psoKvAppend]; setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; [g_enc setBytes:¤t_len length:4 atIndex:5]; [g_enc setBytes:&group_size length:4 atIndex:6]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; - barrier(); return true; } @@ -1397,14 +1212,12 @@ bool cactus_metal_encode_kv_append_i8_m(const void* src, void* int8base, void* s if (!ctx().ok) return false; uint32_t num_groups = (hdim + group_size - 1)/group_size, n = M*kv_heads*num_groups; ensureEncoder(); - g_op="kv"; [g_enc setComputePipelineState:ctx().psoKvAppendM]; setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; [g_enc setBytes:¤t_len length:4 atIndex:5]; [g_enc setBytes:&group_size length:4 atIndex:6]; [g_enc setBytes:&M length:4 atIndex:7]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; - barrier(); return true; } @@ -1414,7 +1227,6 @@ bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, vo if (!ctx().ok) return false; uint32_t num_groups = (hdim + group_size - 1)/group_size, n = M*kv_heads*num_groups; ensureEncoder(); - g_op="kv"; [g_enc setComputePipelineState:ctx().psoKvAppendRingM]; setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; @@ -1422,7 +1234,6 @@ bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, vo [g_enc setBytes:&M length:4 atIndex:7]; [g_enc setBytes:&sink length:4 atIndex:8]; [g_enc setBytes:&W length:4 atIndex:9]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; - barrier(); return true; } @@ -1434,7 +1245,6 @@ bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, v id scrI8 = recycled((size_t)(remaining?remaining:1)*kv_heads*hdim); id scrSc = recycled((size_t)(remaining?remaining:1)*kv_heads*num_groups*sizeof(float)); ensureEncoder(); - g_op="kv"; if (remaining > 0) { uint32_t n = remaining*per; [g_enc setComputePipelineState:ctx().psoSlideS]; @@ -1444,7 +1254,6 @@ bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, v [g_enc setBytes:&group_size length:4 atIndex:6]; [g_enc setBytes:&shift_src length:4 atIndex:7]; [g_enc setBytes:&remaining length:4 atIndex:8]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; - hard_barrier(); } uint32_t n2 = (remaining+1)*per; [g_enc setComputePipelineState:ctx().psoSlideR]; @@ -1454,7 +1263,6 @@ bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, v [g_enc setBytes:&group_size length:4 atIndex:7]; [g_enc setBytes:&keep_sink length:4 atIndex:8]; [g_enc setBytes:&remaining length:4 atIndex:9]; [g_enc dispatchThreads:MTLSizeMake(n2,1,1) threadsPerThreadgroup:MTLSizeMake(n2<256?n2:256,1,1)]; - barrier(); return true; } @@ -1466,7 +1274,6 @@ bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, id scrI8 = recycled((size_t)(remaining?remaining:1)*kv_heads*hdim); id scrSc = recycled((size_t)(remaining?remaining:1)*kv_heads*num_groups*sizeof(float)); ensureEncoder(); - g_op="kv"; if (remaining > 0) { uint32_t n = remaining*per; [g_enc setComputePipelineState:ctx().psoSlideS]; @@ -1476,7 +1283,6 @@ bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, [g_enc setBytes:&group_size length:4 atIndex:6]; [g_enc setBytes:&shift_src length:4 atIndex:7]; [g_enc setBytes:&remaining length:4 atIndex:8]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; - hard_barrier(); } uint32_t n2 = (remaining+M)*per; [g_enc setComputePipelineState:ctx().psoSlideRM]; @@ -1486,7 +1292,6 @@ bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, [g_enc setBytes:&group_size length:4 atIndex:7]; [g_enc setBytes:&keep_sink length:4 atIndex:8]; [g_enc setBytes:&remaining length:4 atIndex:9]; [g_enc setBytes:&M length:4 atIndex:10]; [g_enc dispatchThreads:MTLSizeMake(n2,1,1) threadsPerThreadgroup:MTLSizeMake(n2<256?n2:256,1,1)]; - barrier(); return true; } diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index b351a91b6..9df64d4b8 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -5,19 +5,11 @@ bool cactus_metal_available() { return false; } void cactus_metal_set_active(bool) {} bool cactus_metal_active_mode() { return false; } -bool cactus_metal_concurrent() { return false; } -void cactus_metal_barrier() {} -void cactus_metal_session_set_concurrent(bool) {} -void cactus_metal_group_begin() {} -void cactus_metal_group_end() {} bool cactus_metal_encode_softcap(void*, const void*, size_t, float) { return false; } bool cactus_metal_encode_adjust_logits(void*, size_t, const uint32_t*, uint32_t, const uint32_t*, const float*, uint32_t, int64_t, float) { return false; } -bool cactus_metal_encode_gemv_mega(void* const*, const void*, const void*, float, - const CactusQuantMatrix* const*, int, const void*) { return false; } -bool cactus_metal_encode_gemv_mega_ex(void* const*, const void*, const void*, float, - const CactusQuantMatrix* const*, int, const void*, bool, float) { return false; } +bool cactus_metal_encode_transform_gemv(void*, const void*, const CactusQuantMatrix*, const void*) { return false; } bool cactus_metal_encode_gemv_cat(void* const*, const void* const*, const CactusQuantMatrix* const*, int) { return false; } bool cactus_metal_encode_swiglu_transform(void*, const void*, const void*, diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 78d14153d..f9f61236b 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -297,8 +297,6 @@ def create_parser(): help="Audio file to transcribe (WAV)") transcribe_parser.add_argument("--language", default="en", help="Language code (default: en)") - transcribe_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", - help="Inference backend, default auto (Metal on Apple GPU, gemma-4-E2B-it only)") serve_parser = subparsers.add_parser("serve", help="OpenAI-compatible local HTTP server", parents=[_telemetry_parent(), _build_parent()]) diff --git a/python/cactus/cli/serve.py b/python/cactus/cli/serve.py index 5f587ef9e..67ee60e11 100644 --- a/python/cactus/cli/serve.py +++ b/python/cactus/cli/serve.py @@ -54,6 +54,12 @@ def cmd_serve(args): print_color(RED, "Error: uvicorn not installed. Install the serve extra or run `pip install fastapi uvicorn python-multipart`.") return 1 + backend = getattr(args, "backend", "auto") + if backend and backend != "auto": + from ..bindings.cactus import cactus_set_backend + if cactus_set_backend(backend) != 0: + print_color(YELLOW, f"Backend '{backend}' not available; using auto") + try: from ..server import create_app except ImportError: From c7ae754365c0ffd8fc3a913b88ac3ecb0432eb1e Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 07:19:35 +0200 Subject: [PATCH 12/36] cleanup2 Signed-off-by: jakmro --- apple/CMakeLists.txt | 2 +- cactus-engine/src/constraints.cpp | 9 ++- cactus-engine/src/engine.h | 9 ++- cactus-engine/src/model.cpp | 69 +++++++++++++++-------- cactus-graph/cactus_graph.h | 5 +- cactus-graph/src/execute.cpp | 3 +- cactus-graph/src/gpu_fused.cpp | 47 +++++++++++---- cactus-kernels/metal_backend.h | 3 +- cactus-kernels/src/cactus_kernels.metal | 15 +++-- cactus-kernels/src/metal_backend.mm | 25 ++++---- cactus-kernels/src/metal_backend_stub.cpp | 3 +- 11 files changed, 125 insertions(+), 65 deletions(-) diff --git a/apple/CMakeLists.txt b/apple/CMakeLists.txt index 1afd6078f..e1fb4bbb1 100644 --- a/apple/CMakeLists.txt +++ b/apple/CMakeLists.txt @@ -27,7 +27,7 @@ if(BUILD_SHARED_LIBS) ) target_link_libraries(cactus_framework PUBLIC - "-framework Accelerate" "-framework CoreML" "-framework Foundation" + "-framework Accelerate" "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" ) diff --git a/cactus-engine/src/constraints.cpp b/cactus-engine/src/constraints.cpp index ec6731dbd..c966b7000 100644 --- a/cactus-engine/src/constraints.cpp +++ b/cactus-engine/src/constraints.cpp @@ -124,11 +124,11 @@ void ToolCallConstrainer::mark_trie_bias(const TrieNode* node, char terminator, } if (!has_valid) return; + dense_bias_.assign(valid.size(), TRIE_BLOCK_BIAS); for (size_t token_id = 0; token_id < valid.size(); ++token_id) { - if (!valid[token_id]) { - current_bias_[static_cast(token_id)] = TRIE_BLOCK_BIAS; - } + if (valid[token_id]) dense_bias_[token_id] = 0.0f; } + dense_ready_ = true; } void ToolCallConstrainer::reset_constraint_state() { @@ -432,6 +432,7 @@ void ToolCallConstrainer::init(Config::ModelType model_type, tokenizer_ = tokenizer; generated_text_.clear(); current_bias_.clear(); + dense_ready_ = false; forced_tag_progress_ = 0; const bool model_supported = is_needle() || Config::is_gemma_family(model_type_); active_ = model_supported && !tool_specs_.empty() && tokenizer != nullptr; @@ -520,6 +521,7 @@ void ToolCallConstrainer::update(uint32_t token_id, const std::string& decoded_t void ToolCallConstrainer::compute_bias() { current_bias_.clear(); + dense_ready_ = false; if (!active_) return; @@ -615,6 +617,7 @@ void ToolCallConstrainer::compute_bias() { void ToolCallConstrainer::reset() { generated_text_.clear(); current_bias_.clear(); + dense_ready_ = false; forced_tag_progress_ = 0; reset_constraint_state(); diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 515f4e37a..dfe059329 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -523,6 +523,7 @@ class ToolCallConstrainer { Tokenizer* tokenizer); const std::unordered_map& get_bias() const { return current_bias_; } + const std::vector* get_dense_bias() const { return dense_ready_ ? &dense_bias_ : nullptr; } void update(uint32_t token_id, const std::string& decoded_text); @@ -562,6 +563,8 @@ class ToolCallConstrainer { std::unordered_set close_brace_tokens_; std::unordered_map current_bias_; + std::vector dense_bias_; + bool dense_ready_ = false; void compute_bias(); void tokenize_grammar_elements(); @@ -808,7 +811,7 @@ class Model { bool load_component_graph(Component& comp); void unload_component_graph(Component& comp); bool bind_runtime_buffers(Component& comp); - void run_step(uint32_t token_id, size_t position, bool read_logits); + void run_step(uint32_t token_id, size_t position, bool read_logits, bool use_fused = true); void run_step_batch(const std::vector& token_ids, const std::vector& positions); void set_component_batch(Component& comp, size_t batch); size_t decoder_cache_num_slots(); @@ -947,8 +950,8 @@ class Model { static constexpr size_t MAX_TOKEN_HISTORY = 128; std::vector token_history_; std::vector samp_recent_; - std::vector samp_bias_ids_; - std::vector samp_bias_vals_; + std::vector samp_bias_dense_; + bool samp_has_bias_ = false; float samp_penalty_ = 1.0f; bool samp_ctx_active_ = false; std::mt19937 sample_rng_{std::random_device{}()}; diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 07c1efb26..2331dbbd3 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -895,16 +895,19 @@ void Model::copy_encoder_outputs_to_decoder(const Component& enc) { } } -void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { +void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/, bool use_fused) { if (decode_route_ == DecodeRoute::DIRECT_DECODER_STEP) { write_int_input(*decoder_, "input_ids", static_cast(token_id)); write_int_input(*decoder_, "position_ids", static_cast(position)); - const bool fused = cactus_backend_fused(); + const bool fused = use_fused && cactus_backend_fused(); + if (!fused) cactus_graph_set_prefill_consistent(true); if (!fused || !decoder_->graph->execute_gpu_fused()) decoder_->graph->execute(); + cactus_graph_set_prefill_consistent(false); maybe_capture_handoff_probe_hidden(*decoder_); return; } - const bool fused = cactus_backend_fused(); + if (!use_fused) cactus_graph_set_prefill_consistent(true); + const bool fused = use_fused && cactus_backend_fused(); if (fused) { if (ple_probe_state_ == 0) ple_probe_state_ = encoder_->graph->extract_ple_pathway(fused_embed_ctx_) ? 1 : 2; @@ -914,12 +917,13 @@ void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { cactus_graph_set_fused_embed(&fused_embed_ctx_); bool ok = decoder_->graph->execute_gpu_fused(); cactus_graph_set_fused_embed(nullptr); - if (ok) { maybe_capture_handoff_probe_hidden(*decoder_); return; } + if (ok) { cactus_graph_set_prefill_consistent(false); maybe_capture_handoff_probe_hidden(*decoder_); return; } } } run_encoder_step(token_id, position); copy_component_outputs_to_inputs(*encoder_, *decoder_); if (!fused || !decoder_->graph->execute_gpu_fused()) decoder_->graph->execute(); + cactus_graph_set_prefill_consistent(false); maybe_capture_handoff_probe_hidden(*decoder_); } @@ -1946,15 +1950,17 @@ uint32_t Model::argmax_logits_at(const BufferDesc& desc, void* ptr, size_t row_o float best_v = -std::numeric_limits::infinity(); float second_v = -std::numeric_limits::infinity(); const auto& tool_bias = tool_constrainer_.get_bias(); + const std::vector* tool_dense = tool_constrainer_.get_dense_bias(); { uint32_t gidx; float gbest, gsecond; - if (tool_bias.empty() && vocab_bias_.empty() && suppressed_token_id_ < 0 && + if (!tool_dense && tool_bias.empty() && vocab_bias_.empty() && suppressed_token_id_ < 0 && cactus_graph_gpu_argmax(&gidx, &gbest, &gsecond)) { if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(gbest, gsecond); return gidx; } } auto score_with_bias = [&](size_t token_id, float value) { + if (tool_dense && token_id < tool_dense->size()) value += (*tool_dense)[token_id]; auto tool_it = tool_bias.find(static_cast(token_id)); if (tool_it != tool_bias.end()) value += tool_it->second; auto vocab_it = vocab_bias_.find(static_cast(token_id)); @@ -2024,8 +2030,7 @@ uint32_t Model::argmax_last_logits(float* out_uncertainty) { void Model::prepare_sampling_context(float repetition_penalty) { samp_recent_.clear(); - samp_bias_ids_.clear(); - samp_bias_vals_.clear(); + samp_has_bias_ = false; samp_penalty_ = repetition_penalty; constexpr size_t kPenaltyWindow = 64; size_t start = token_history_.size() > kPenaltyWindow ? token_history_.size() - kPenaltyWindow : 0; @@ -2035,13 +2040,28 @@ void Model::prepare_sampling_context(float repetition_penalty) { for (uint32_t r : samp_recent_) if (r == id) { dup = true; break; } if (!dup) samp_recent_.push_back(id); } - for (const auto& kv : tool_constrainer_.get_bias()) { samp_bias_ids_.push_back(kv.first); samp_bias_vals_.push_back(kv.second); } - for (const auto& kv : vocab_bias_) { samp_bias_ids_.push_back(kv.first); samp_bias_vals_.push_back(kv.second); } + const std::vector* tool_dense = tool_constrainer_.get_dense_bias(); + const auto& tool_bias = tool_constrainer_.get_bias(); + if (tool_dense || !tool_bias.empty() || !vocab_bias_.empty()) { + size_t n = tokenizer_ ? tokenizer_->get_vocab_size() : 0; + if (tool_dense && tool_dense->size() > n) n = tool_dense->size(); + if (n > 0) { + if (tool_dense) samp_bias_dense_.assign(tool_dense->begin(), tool_dense->end()); + else samp_bias_dense_.assign(n, 0.0f); + if (samp_bias_dense_.size() < n) samp_bias_dense_.resize(n, 0.0f); + for (const auto& kv : tool_bias) + if (kv.first < samp_bias_dense_.size()) samp_bias_dense_[kv.first] += kv.second; + for (const auto& kv : vocab_bias_) + if (kv.first < samp_bias_dense_.size()) samp_bias_dense_[kv.first] += kv.second; + samp_has_bias_ = true; + } + } samp_ctx_active_ = (repetition_penalty != 1.0f && !samp_recent_.empty()) || - !samp_bias_ids_.empty() || suppressed_token_id_ >= 0; + samp_has_bias_ || suppressed_token_id_ >= 0; if (samp_ctx_active_) { cactus_graph_set_sampling(samp_recent_.data(), (int)samp_recent_.size(), repetition_penalty, - samp_bias_ids_.data(), samp_bias_vals_.data(), (int)samp_bias_ids_.size(), + samp_has_bias_ ? samp_bias_dense_.data() : nullptr, + samp_has_bias_ ? samp_bias_dense_.size() : 0, suppressed_token_id_); } else { cactus_graph_clear_sampling(); @@ -2077,25 +2097,30 @@ uint32_t Model::sample_component_logits(Component& comp, float temperature, floa put(id, v > 0.0f ? v / samp_penalty_ : v * samp_penalty_); } } - for (size_t i = 0; i < samp_bias_ids_.size(); ++i) { - uint32_t id = samp_bias_ids_[i]; - if (id < vocab) put(id, get(id) + samp_bias_vals_[i]); - } if (suppressed_token_id_ >= 0 && (size_t)suppressed_token_id_ < vocab) put((size_t)suppressed_token_id_, -65504.0f); } - if (greedy) { uint32_t gidx; float gbest, gsecond; - if ((!samp_ctx_active_ || cactus_graph_gpu_adjusted()) && + if ((!samp_has_bias_ || cactus_graph_gpu_argmax_biased()) && cactus_graph_gpu_argmax(&gidx, &gbest, &gsecond)) { if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(gbest, gsecond); return gidx; } + } + const float* bd = samp_has_bias_ ? samp_bias_dense_.data() : nullptr; + const size_t bn = samp_has_bias_ ? samp_bias_dense_.size() : 0; + auto biased = [&](size_t i) -> float { + float v = get(i); + if (i < bn) v += bd[i]; + return v; + }; + + if (greedy) { uint32_t best = 0; float bv = -std::numeric_limits::infinity(), sv = bv; for (size_t i = 0; i < vocab; ++i) { - float v = get(i); + float v = biased(i); if (v > bv) { sv = bv; bv = v; best = (uint32_t)i; } else if (v > sv) sv = v; } @@ -2108,7 +2133,7 @@ uint32_t Model::sample_component_logits(Component& comp, float temperature, floa cand.reserve(2 * K + 16); float kmin = -std::numeric_limits::infinity(); for (size_t i = 0; i < vocab; ++i) { - float v = get(i); + float v = biased(i); if (v <= kmin) continue; cand.emplace_back(v, (uint32_t)i); if (cand.size() >= 2 * K) { @@ -2213,7 +2238,7 @@ bool Model::prefill_and_sample_first_token(const std::vector& tokens, // Cache already moved into the step component; drop the padded last row and re-run it for logits. set_cache_current_len(*decoder_, tokens.size() - 1); cache_total_seq_len_ = tokens.size() - 1; - run_step(tokens.back(), cache_total_seq_len_, true); + run_step(tokens.back(), cache_total_seq_len_, true, /*use_fused=*/false); ++cache_total_seq_len_; out_token = argmax_last_logits(out_uncertainty); record_sampled_token(out_token); @@ -2224,7 +2249,7 @@ bool Model::prefill_and_sample_first_token(const std::vector& tokens, cache_total_seq_len_ += chunked.logical_tokens; } for (size_t i = chunked.logical_tokens; i < tokens.size(); ++i) { - run_step(tokens[i], cache_total_seq_len_, i + 1 == tokens.size()); + run_step(tokens[i], cache_total_seq_len_, i + 1 == tokens.size(), /*use_fused=*/false); ++cache_total_seq_len_; } last_prefill_scalar_tail_tokens_ = tokens.size() - chunked.logical_tokens; @@ -2255,7 +2280,7 @@ void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, ChunkedPrefillResult chunked = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), prepare_decode); cache_total_seq_len_ += chunked.logical_tokens; for (size_t i = chunked.logical_tokens; i < tokens.size(); ++i) { - run_step(tokens[i], cache_total_seq_len_, /*read_logits=*/false); + run_step(tokens[i], cache_total_seq_len_, /*read_logits=*/false, /*use_fused=*/false); ++cache_total_seq_len_; } cache_token_ids_.insert(cache_token_ids_.end(), tokens.begin(), tokens.end()); diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 80a318605..b026abaa1 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -468,12 +468,15 @@ void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx); bool cactus_graph_gpu_argmax(uint32_t* idx, float* best, float* second); void cactus_graph_mark_unadjusted(); +void cactus_graph_set_prefill_consistent(bool on); +bool cactus_graph_prefill_consistent(); void cactus_graph_on_destroy(const void* graph); void cactus_graph_set_sampling(const uint32_t* recent, int n_recent, float rep_penalty, - const uint32_t* bias_ids, const float* bias_vals, int n_bias, + const float* bias_dense, size_t bias_len, long long suppressed); void cactus_graph_clear_sampling(); bool cactus_graph_gpu_adjusted(); +bool cactus_graph_gpu_argmax_biased(); class CactusGraph { public: diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index cc43d6adc..f575781df 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -472,7 +472,8 @@ static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const nod if (!(PrecisionTraits::is_cq(rhs.precision) && rhs.group_size > 0)) return false; if (!fp16(lhs)) return false; CactusQuantMatrix mat = rhs.to_cq_matrix(); - if (M == 1) return cactus_metal_encode_quant_matmul(out.get_data(), lhs.get_data(), &mat); + if (M == 1 && !cactus_graph_prefill_consistent()) + return cactus_metal_encode_quant_matmul(out.get_data(), lhs.get_data(), &mat); return cactus_metal_encode_quant_matmul_m(out.get_data(), lhs.get_data(), &mat, (uint32_t)M); } case OpType::ADD: case OpType::ADD_CLIPPED: case OpType::SUBTRACT: diff --git a/cactus-graph/src/gpu_fused.cpp b/cactus-graph/src/gpu_fused.cpp index 0192fdeb4..2a2848f99 100644 --- a/cactus-graph/src/gpu_fused.cpp +++ b/cactus-graph/src/gpu_fused.cpp @@ -61,39 +61,62 @@ static const float* g_gpu_argmax_buf = nullptr; } static bool g_last_step_adjusted = false; +static bool g_last_amax_biased = false; +static bool g_prefill_consistent = false; void cactus_graph_mark_unadjusted() { g_last_step_adjusted = false; + g_last_amax_biased = false; + g_gpu_argmax_valid = false; } +void cactus_graph_set_prefill_consistent(bool on) { g_prefill_consistent = on; } +bool cactus_graph_prefill_consistent() { return g_prefill_consistent; } + struct GSampling { bool active = false; float rep_penalty = 1.0f; std::vector recent; - std::vector bias_ids; - std::vector bias_vals; + const float* bias_dense = nullptr; + size_t bias_len = 0; long long suppressed = -1; }; static GSampling g_samp; static bool g_step_adjusted = false; +static bool g_step_amax_biased = false; +static float* g_bias_dense = nullptr; +static size_t g_bias_dense_n = 0; void cactus_graph_set_sampling(const uint32_t* recent, int n_recent, float rep_penalty, - const uint32_t* bias_ids, const float* bias_vals, int n_bias, + const float* bias_dense, size_t bias_len, long long suppressed) { g_samp.active = true; g_samp.rep_penalty = rep_penalty; g_samp.recent.assign(recent, recent + (n_recent > 0 ? n_recent : 0)); - g_samp.bias_ids.assign(bias_ids, bias_ids + (n_bias > 0 ? n_bias : 0)); - g_samp.bias_vals.assign(bias_vals, bias_vals + (n_bias > 0 ? n_bias : 0)); + g_samp.bias_dense = bias_dense; + g_samp.bias_len = bias_dense ? bias_len : 0; g_samp.suppressed = suppressed; } -void cactus_graph_clear_sampling() { g_samp.active = false; } +void cactus_graph_clear_sampling() { g_samp = GSampling(); } bool cactus_graph_gpu_adjusted() { return g_last_step_adjusted; } +bool cactus_graph_gpu_argmax_biased() { return g_last_amax_biased; } + +static const float* update_bias_dense(size_t vocab) { + if (!g_samp.bias_dense || g_samp.bias_len == 0) return nullptr; + if (!g_bias_dense || g_bias_dense_n < vocab) { + g_bias_dense = (float*)cactus_metal_alloc_shared(vocab * sizeof(float)); + if (!g_bias_dense) return nullptr; + g_bias_dense_n = vocab; + } + size_t n = g_samp.bias_len < vocab ? g_samp.bias_len : vocab; + std::memcpy(g_bias_dense, g_samp.bias_dense, n * sizeof(float)); + if (n < vocab) std::memset(g_bias_dense + n, 0, (vocab - n) * sizeof(float)); + return g_bias_dense; +} static inline bool sampling_needs_adjust() { return g_samp.active && - ((g_samp.rep_penalty != 1.0f && !g_samp.recent.empty()) || - !g_samp.bias_ids.empty() || g_samp.suppressed >= 0); + ((g_samp.rep_penalty != 1.0f && !g_samp.recent.empty()) || g_samp.suppressed >= 0); } bool cactus_graph_gpu_argmax(uint32_t* idx, float* best, float* second) { @@ -435,12 +458,13 @@ bool CactusGraph::execute_gpu_fused() { if (sampling_needs_adjust()) { adjusted = cactus_metal_encode_adjust_logits(lg, V, g_samp.recent.data(), (uint32_t)g_samp.recent.size(), - g_samp.bias_ids.data(), g_samp.bias_vals.data(), (uint32_t)g_samp.bias_ids.size(), g_samp.suppressed, g_samp.rep_penalty); if (!adjusted) return 0; } g_step_adjusted = adjusted; - cactus_metal_encode_argmax(lg, (uint32_t)V, G.amax); + const float* bias = g_samp.active ? update_bias_dense(V) : nullptr; + if (!cactus_metal_encode_argmax(lg, (uint32_t)V, G.amax, bias)) return 0; + g_step_amax_biased = (bias != nullptr); return 1; } cactus_metal_session_sync(); @@ -463,9 +487,11 @@ bool CactusGraph::execute_gpu_fused() { g_gpu_argmax_buf = (const float*)G.amax; g_gpu_argmax_valid = true; g_last_step_adjusted = g_step_adjusted; + g_last_amax_biased = g_step_amax_biased; } else { B(3860).set_external(lg); g_last_step_adjusted = false; + g_last_amax_biased = false; } return true; } @@ -473,6 +499,7 @@ bool CactusGraph::execute_gpu_fused() { void cactus_graph_on_destroy(const void* graph) { if (G.init && G.owner == graph) G.init = false; + g_samp = GSampling(); cactus_metal_invalidate_host_wraps(); } diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 14e687c42..68f508e4e 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -52,11 +52,10 @@ bool cactus_metal_encode_rms_norm_add_rms(void* h_out, void* xn_out, const void* size_t rows, size_t dim, float eps, float out_scale); bool cactus_metal_encode_rms_norm_add_scale(void* out, const void* in, const void* weight, const void* res, size_t rows, size_t dim, float eps, float out_scale); -bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3); +bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3, const void* bias); bool cactus_metal_encode_softcap(void* out, const void* in, size_t n, float cap); bool cactus_metal_encode_adjust_logits(void* logits, size_t vocab, const uint32_t* recent, uint32_t n_recent, - const uint32_t* bias_ids, const float* bias_vals, uint32_t n_bias, int64_t suppressed, float penalty); bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_prec, size_t n); bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W); diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 3e1b2fae3..a27dd6907 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -2051,26 +2051,22 @@ kernel void kv_append_ring_i8_m(device const half* src [[buffer(0)]], device cha for (uint k=0;k= U.vocab) continue; float v = (float)logits[id]; logits[id] = (half)(v > 0.0f ? v/U.penalty : v*U.penalty); } } threadgroup_barrier(mem_flags::mem_device); - for (uint i=t; i b) { s = b; b = v; bi = i; } else if (v > s) { s = v; } } diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 73c9c249e..e05337818 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -462,18 +462,22 @@ bool cactus_metal_encode_rms_norm_add_rms(void* h_out, void* xn_out, const void* return true; } -bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) { +bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3, const void* bias) { if (!ctx().ok) return false; + if (bias && !(ctx().psoArgmaxP && ctx().psoArgmaxF && vocab >= 32768u)) return false; ensureEncoder(); uint32_t V = vocab; if (ctx().psoArgmaxP && ctx().psoArgmaxF && V >= 32768u) { const uint32_t NP = 128u, T = 256u; uint32_t chunk = (V + NP - 1u)/NP; + uint32_t has_bias = bias ? 1u : 0u; id part = recycled((size_t)NP*3*sizeof(float)); [g_enc setComputePipelineState:ctx().psoArgmaxP]; setBufAt(logits, (size_t)vocab*2, 0); [g_enc setBuffer:part offset:0 atIndex:1]; [g_enc setBytes:&V length:4 atIndex:2]; [g_enc setBytes:&chunk length:4 atIndex:3]; + if (bias) setBufAt(bias, (size_t)vocab*sizeof(float), 4); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:4]; + [g_enc setBytes:&has_bias length:4 atIndex:5]; [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:0]; [g_enc setThreadgroupMemoryLength:T*sizeof(uint) atIndex:1]; [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:2]; @@ -501,23 +505,20 @@ bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3) } bool cactus_metal_encode_adjust_logits(void* logits, size_t vocab, const uint32_t* recent, uint32_t n_recent, - const uint32_t* bias_ids, const float* bias_vals, uint32_t n_bias, int64_t suppressed, float penalty) { if (!ctx().ok || !ctx().psoAdjust) return false; - if (n_recent > 4096u || n_bias > 65536u) return false; + if (n_recent > 4096u) return false; ensureEncoder(); - id rb = ctx().dummy, bi = ctx().dummy, bv = ctx().dummy; + id rb = ctx().dummy; if (n_recent) { rb = recycled((size_t)n_recent*4); std::memcpy([rb contents], recent, (size_t)n_recent*4); } - if (n_bias) { bi = recycled((size_t)n_bias*4); std::memcpy([bi contents], bias_ids, (size_t)n_bias*4); - bv = recycled((size_t)n_bias*4); std::memcpy([bv contents], bias_vals, (size_t)n_bias*4); } - struct { uint32_t n_recent, n_bias, suppress_flag, suppress_id; float penalty; } U = - { n_recent, n_bias, - (uint32_t)(suppressed >= 0 ? 1 : 0), (uint32_t)(suppressed >= 0 ? suppressed : 0), penalty }; + struct { uint32_t n_recent, suppress_flag, suppress_id, vocab_n; float penalty; } U = + { n_recent, + (uint32_t)(suppressed >= 0 ? 1 : 0), (uint32_t)(suppressed >= 0 ? suppressed : 0), + (uint32_t)vocab, penalty }; [g_enc setComputePipelineState:ctx().psoAdjust]; setBufAt(logits, vocab*2, 0); [g_enc setBuffer:rb offset:0 atIndex:1]; - [g_enc setBuffer:bi offset:0 atIndex:2]; [g_enc setBuffer:bv offset:0 atIndex:3]; - [g_enc setBytes:&U length:sizeof(U) atIndex:4]; + [g_enc setBytes:&U length:sizeof(U) atIndex:2]; [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(1024,1,1)]; return true; } @@ -741,7 +742,7 @@ bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { } bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M) { - if (M == 1) return cactus_metal_encode_quant_matmul(out, lhs, W); + const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; const uint32_t bits=W->bits, pgb=(gs*bits+7u)/8u; const bool fast = ctx().ok && quant_fast_eligible(W); diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 9df64d4b8..14fb056e6 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -7,7 +7,6 @@ void cactus_metal_set_active(bool) {} bool cactus_metal_active_mode() { return false; } bool cactus_metal_encode_softcap(void*, const void*, size_t, float) { return false; } bool cactus_metal_encode_adjust_logits(void*, size_t, const uint32_t*, uint32_t, - const uint32_t*, const float*, uint32_t, int64_t, float) { return false; } bool cactus_metal_encode_transform_gemv(void*, const void*, const CactusQuantMatrix*, const void*) { return false; } bool cactus_metal_encode_gemv_cat(void* const*, const void* const*, @@ -38,7 +37,7 @@ bool cactus_metal_encode_rms_norm_add(void*, const void*, const void*, const voi bool cactus_metal_encode_rms_norm_add_rms(void*, void*, const void*, const void*, const void*, const void*, size_t, size_t, float, float) { return false; } bool cactus_metal_encode_rms_norm_add_scale(void*, const void*, const void*, const void*, size_t, size_t, float, float) { return false; } -bool cactus_metal_encode_argmax(const void*, uint32_t, void*) { return false; } +bool cactus_metal_encode_argmax(const void*, uint32_t, void*, const void*) { return false; } bool cactus_metal_encode_cast(void*, int, const void*, int, size_t) { return false; } bool cactus_metal_encode_quant_matmul(void*, const void*, const CactusQuantMatrix*) { return false; } bool cactus_metal_encode_quant_matmul_m(void*, const void*, const CactusQuantMatrix*, uint32_t) { return false; } From bc14911b626ca94cd676886e47850f6ae637c52c Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 10:08:11 +0200 Subject: [PATCH 13/36] Metal execution for vision and audio encoder towers Whisper audio embed 40.9ms vs CoreML 54.2ms, LFM2-VL vision embed 61.7ms vs CoreML 77.6ms, gemma-4 towers 3-5x faster than CPU. Adds dense fp16 GEMM via MPS with in-encoder fallback, flash attention for fp16 K/V with mask/GQA/logit-cap support, layer norm, softmax, GLU, clamp, conv1d_k3 and fp32 elementwise kernels; generic executor gains an fp16 retype pass for cast-bounded fp32 regions, GPU-dirty lazy sync, periodic mid-graph flush, and dense fp16 matmul/attention/layernorm/conv encode paths. Tower graphs stay resident on the Metal backend, vDSP FFT setups are cached, and lfm2 image embeddings resolve projector output names. Signed-off-by: jakmro --- apple/CMakeLists.txt | 2 +- cactus-engine/CMakeLists.txt | 4 +- cactus-engine/src/model.cpp | 23 +- cactus-graph/cactus_graph.h | 4 + cactus-graph/src/execute.cpp | 238 +++++++++++++++- cactus-kernels/CMakeLists.txt | 2 +- cactus-kernels/metal_backend.h | 16 ++ cactus-kernels/src/cactus_kernels.metal | 324 ++++++++++++++++++++++ cactus-kernels/src/dsp.cpp | 13 +- cactus-kernels/src/metal_backend.mm | 229 ++++++++++++++- cactus-kernels/src/metal_backend_stub.cpp | 13 + python/cactus/cli/compile.py | 1 + python/cactus/cli/runtime.py | 1 + 13 files changed, 841 insertions(+), 29 deletions(-) diff --git a/apple/CMakeLists.txt b/apple/CMakeLists.txt index e1fb4bbb1..2fabddd8c 100644 --- a/apple/CMakeLists.txt +++ b/apple/CMakeLists.txt @@ -27,7 +27,7 @@ if(BUILD_SHARED_LIBS) ) target_link_libraries(cactus_framework PUBLIC - "-framework Accelerate" "-framework CoreML" "-framework Foundation" "-framework Metal" + "-framework Accelerate" "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders" "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" ) diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index 58e87cb86..7e057285e 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -56,7 +56,7 @@ target_link_libraries(cactus_engine PUBLIC cactus_graph) if(APPLE) target_link_libraries(cactus_engine PUBLIC - "-framework CoreML" "-framework Foundation" "-framework Metal" + "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders" "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" ) @@ -156,7 +156,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) target_link_libraries(cactus_engine_shared PRIVATE "${_curl}") endif() target_link_libraries(cactus_engine_shared PRIVATE - "-framework Accelerate" "-framework CoreML" "-framework Foundation" "-framework Metal" + "-framework Accelerate" "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders" "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" ) else() diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 2331dbbd3..35c41895c 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -808,6 +808,7 @@ bool Model::load_component_graph(Component& comp) { } void Model::unload_component_graph(Component& comp) { + if (cactus_backend_metal()) return; if (comp.graph) { comp.graph->release_runtime_buffers(); comp.graph->release_all_weight_pages(); @@ -1615,8 +1616,10 @@ void Model::run_vision_encoder(const std::string& image_path) { media_feature_shapes_[name] = desc.shape; media_feature_precisions_[name] = desc.precision; } - vision_encoder_->graph->release_runtime_buffers(); - vision_encoder_->graph->release_all_weight_pages(); + if (!cactus_backend_metal()) { + vision_encoder_->graph->release_runtime_buffers(); + vision_encoder_->graph->release_all_weight_pages(); + } unload_component_graph(*vision_encoder_); } @@ -1830,8 +1833,10 @@ void Model::run_audio_encoder_messages(const std::vector>& au if (mel.empty()) continue; run_audio_encoder(mel); } - audio_encoder_->graph->release_runtime_buffers(); - audio_encoder_->graph->release_all_weight_pages(); + if (!cactus_backend_metal()) { + audio_encoder_->graph->release_runtime_buffers(); + audio_encoder_->graph->release_all_weight_pages(); + } unload_component_graph(*audio_encoder_); } @@ -3687,10 +3692,15 @@ std::vector Model::get_image_embeddings(const std::string& image_path) { if (vision_encoder_->logical_outputs.empty()) { throw std::runtime_error("vision_encoder has no logical outputs"); } - const std::string output_name = vision_encoder_->logical_outputs[0]; + std::string output_name = vision_encoder_->logical_outputs[0]; run_vision_encoder(image_path); + if (!media_features_.count(output_name)) { + for (const char* name : {"image_features", "image_embeddings", "vision_features"}) { + if (media_features_.count(name)) { output_name = name; break; } + } + } auto bytes_it = media_features_.find(output_name); auto shape_it = media_feature_shapes_.find(output_name); auto prec_it = media_feature_precisions_.find(output_name); @@ -3707,6 +3717,9 @@ std::vector Model::get_image_embeddings(const std::string& image_path) { media_feature_shapes_.erase(name); media_feature_precisions_.erase(name); } + media_features_.erase(output_name); + media_feature_shapes_.erase(output_name); + media_feature_precisions_.erase(output_name); // run_vision_encoder unloads the graph; restore so subsequent paths that // assume the encoder is loaded (e.g. transcribe_*) keep working. load_component_graph(*vision_encoder_); diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index b026abaa1..3e4e2f6c7 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -732,6 +732,7 @@ class CactusGraph { void execute(const std::string& profile_file = ""); bool execute_gpu_fused(); + void build_gpu_retype_plan(); bool extract_ple_pathway(FusedEmbedCtx& ctx) const; void hard_reset(); void soft_reset(); @@ -785,6 +786,9 @@ class CactusGraph { std::unordered_set populated_node_ids_; std::unordered_set embedded_input_node_ids_; std::unordered_set retained_output_node_ids_; + std::vector gpu_retype_plan_; + bool gpu_retype_built_ = false; + bool gpu_retype_disabled_ = false; std::unordered_map> gpu_persistent_acts_; }; diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index f575781df..5ba9bdbeb 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -469,20 +469,37 @@ static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const nod const auto& lhs = get_input(node, 0, nodes, map); const auto& rhs = get_input(node, 1, nodes, map); size_t M = lhs.shape[lhs.shape.size() - 2]; - if (!(PrecisionTraits::is_cq(rhs.precision) && rhs.group_size > 0)) return false; if (!fp16(lhs)) return false; - CactusQuantMatrix mat = rhs.to_cq_matrix(); - if (M == 1 && !cactus_graph_prefill_consistent()) - return cactus_metal_encode_quant_matmul(out.get_data(), lhs.get_data(), &mat); - return cactus_metal_encode_quant_matmul_m(out.get_data(), lhs.get_data(), &mat, (uint32_t)M); + if (PrecisionTraits::is_cq(rhs.precision) && rhs.group_size > 0) { + CactusQuantMatrix mat = rhs.to_cq_matrix(); + if (M == 1 && !cactus_graph_prefill_consistent()) + return cactus_metal_encode_quant_matmul(out.get_data(), lhs.get_data(), &mat); + return cactus_metal_encode_quant_matmul_m(out.get_data(), lhs.get_data(), &mat, (uint32_t)M); + } + if (!fp16(rhs) || !fp16(out) || rhs.shape.size() != 2) return false; + size_t K = lhs.shape.back(); + size_t N = node.params.pretransposed_rhs ? rhs.shape[0] : rhs.shape[1]; + size_t rhs_k = node.params.pretransposed_rhs ? rhs.shape[1] : rhs.shape[0]; + if (K == 0 || rhs_k != K || lhs.total_size % K != 0) return false; + size_t rows = lhs.total_size / K; + if (rows != M) { + size_t batch = rows / M; + if (batch * M != rows) return false; + } + return cactus_metal_encode_gemm_f16(out.get_data(), lhs.get_data(), rhs.get_data(), + (uint32_t)rows, (uint32_t)K, (uint32_t)N, node.params.pretransposed_rhs ? 1 : 0); } case OpType::ADD: case OpType::ADD_CLIPPED: case OpType::SUBTRACT: case OpType::MULTIPLY: case OpType::DIVIDE: { const auto& a = get_input(node, 0, nodes, map); const auto& b = get_input(node, 1, nodes, map); - if (!fp16(a) || !fp16(b) || !fp16(out)) return false; int code = node.op_type==OpType::ADD?0: node.op_type==OpType::ADD_CLIPPED?1: node.op_type==OpType::SUBTRACT?2: node.op_type==OpType::MULTIPLY?3:4; + bool f32 = a.precision == Precision::FP32 && b.precision == Precision::FP32 + && out.precision == Precision::FP32; + if (f32 && a.total_size == out.total_size && b.total_size == out.total_size) + return cactus_metal_encode_binary_f32(code, out.get_data(), a.get_data(), b.get_data(), out.total_size); + if (!fp16(a) || !fp16(b) || !fp16(out)) return false; if (a.total_size == out.total_size && b.total_size == out.total_size) return cactus_metal_encode_binary(code, out.get_data(), a.get_data(), b.get_data(), out.total_size); const auto& osh = out.shape; @@ -552,17 +569,98 @@ static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const nod case OpType::SCALAR_ADD: case OpType::SCALAR_SUBTRACT: case OpType::SCALAR_MULTIPLY: case OpType::SCALAR_DIVIDE: { const auto& in = get_input(node, 0, nodes, map); - if (!fp16(in) || !fp16(out)) return false; int code = node.op_type==OpType::SCALAR_ADD?0: node.op_type==OpType::SCALAR_SUBTRACT?1: node.op_type==OpType::SCALAR_MULTIPLY?2:3; + if (in.precision == Precision::FP32 && out.precision == Precision::FP32) + return cactus_metal_encode_scalar_f32(code, out.get_data(), in.get_data(), out.total_size, node.params.scalar); + if (!fp16(in) || !fp16(out)) return false; return cactus_metal_encode_scalar(code, out.get_data(), in.get_data(), out.total_size, node.params.scalar); } case OpType::GELU: case OpType::TANH: case OpType::SILU: case OpType::RELU: { const auto& in = get_input(node, 0, nodes, map); - if (!fp16(in) || !fp16(out)) return false; int code = node.op_type==OpType::GELU?0: node.op_type==OpType::TANH?1: node.op_type==OpType::SILU?2:3; + if (in.precision == Precision::FP32 && out.precision == Precision::FP32) + return cactus_metal_encode_unary_f32(code, out.get_data(), in.get_data(), out.total_size); + if (!fp16(in) || !fp16(out)) return false; return cactus_metal_encode_unary(code, out.get_data(), in.get_data(), out.total_size); } + case OpType::CLAMP: { + const auto& in = get_input(node, 0, nodes, map); + bool f32 = in.precision == Precision::FP32 && out.precision == Precision::FP32; + if (!f32 && (!fp16(in) || !fp16(out))) return false; + return cactus_metal_encode_clamp(out.get_data(), in.get_data(), out.total_size, + node.params.scalar, node.params.scale, f32 ? 1 : 0); + } + case OpType::GLU: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || !fp16(out) || out.shape.empty()) return false; + int axis = node.params.axis; + if (axis < 0) axis += (int)in.shape.size(); + if (axis < 0 || (size_t)axis >= in.shape.size()) return false; + size_t split = in.shape[(size_t)axis] / 2; + size_t inner = 1; + for (size_t i = (size_t)axis + 1; i < in.shape.size(); ++i) inner *= in.shape[i]; + return cactus_metal_encode_glu(out.get_data(), in.get_data(), split, inner, out.total_size); + } + case OpType::SOFTMAX: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || !fp16(out) || out.shape.empty()) return false; + size_t cols = out.shape.back(); + if (cols == 0) return false; + return cactus_metal_encode_softmax_rows(out.get_data(), in.get_data(), out.total_size / cols, cols); + } + case OpType::LAYERNORM: { + if (node.input_ids.size() < 2 || out.shape.empty()) return false; + const auto& in = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const BufferDesc* b = node.input_ids.size() > 2 ? &get_input(node, 2, nodes, map) : nullptr; + if (!fp16(in) || !fp16(out) || !fp16(w) || (b && b->precision != Precision::FP16)) return false; + size_t dim = out.shape.back(); + if (dim == 0 || w.total_size != dim) return false; + return cactus_metal_encode_layer_norm(out.get_data(), in.get_data(), w.get_data(), + b ? b->get_data() : nullptr, out.total_size / dim, dim, node.params.epsilon); + } + case OpType::CONV1D_K3: { + if (node.input_ids.size() < 2) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + if (!fp16(x) || !fp16(out)) return false; + if (x.shape.size() != 3 || w.shape.size() != 3 || x.shape[0] != 1 || w.shape[2] != 3) return false; + size_t Cin = x.shape[1], L = x.shape[2], Cout = w.shape[0]; + size_t stride = node.params.stride ? node.params.stride : 1; + size_t Lout = ((L - 1) / stride) + 1; + int w_int8 = w.precision == Precision::INT8 ? 1 : 0; + if (!w_int8 && w.precision != Precision::FP16) return false; + if (out.total_size != Cout * Lout) return false; + return cactus_metal_encode_conv1d_k3(out.get_data(), x.get_data(), w.get_data(), w_int8, + w.activation_scales_data, (uint32_t)w.group_size, + (uint32_t)Cin, (uint32_t)L, (uint32_t)Cout, (uint32_t)Lout, (uint32_t)stride); + } + case OpType::ATTENTION: { + if (node.input_ids.size() < 3) return false; + const auto& q = get_input(node, 0, nodes, map); + const auto& k = get_input(node, 1, nodes, map); + const auto& v = get_input(node, 2, nodes, map); + const BufferDesc* mk = node.input_ids.size() > 3 ? &get_input(node, 3, nodes, map) : nullptr; + if (!fp16(q) || !fp16(k) || !fp16(v) || !fp16(out)) return false; + if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) return false; + size_t B = q.shape[0], T = q.shape[1], HQ = q.shape[2], D = q.shape[3]; + size_t S = k.shape[1], HKV = k.shape[2], DV = v.shape[3]; + if (HKV == 0 || HQ % HKV != 0) return false; + uint32_t mask_mode = 0; + if (mk) { + if (mk->precision != Precision::FP16) return false; + bool per_head = mk->shape.size() == 4; + if (!per_head && mk->shape.size() != 3) return false; + mask_mode = (node.params.attention_mask_is_additive ? 1u : 2u) + (per_head ? 2u : 0u); + } + float scale = node.params.scale != 0.0f ? node.params.scale : 1.0f/std::sqrt((float)D); + return cactus_metal_encode_attention_f16(out.get_data(), q.get_data(), k.get_data(), v.get_data(), + mk ? mk->get_data() : nullptr, + (uint32_t)B, (uint32_t)T, (uint32_t)S, (uint32_t)HQ, (uint32_t)HKV, (uint32_t)D, (uint32_t)DV, + scale, node.params.is_causal ? 1u : 0u, (uint32_t)node.params.position_offset, + (uint32_t)node.params.window_size, node.params.logit_cap, mask_mode); + } case OpType::RMS_NORM: { if (node.input_ids.size() < 2 || out.shape.empty()) return false; const auto& in = get_input(node, 0, nodes, map); @@ -748,6 +846,76 @@ static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const nod } } +void CactusGraph::build_gpu_retype_plan() { + gpu_retype_built_ = true; + const size_t n = nodes_.size(); + gpu_retype_plan_.assign(n, 0); + auto idxof = [&](size_t id) -> long { + auto it = node_index_map_.find(id); + return it == node_index_map_.end() ? -1 : (long)it->second; + }; + std::vector> cons(n); + for (size_t i = 0; i < n; ++i) + for (size_t id : nodes_[i]->input_ids) { long j = idxof(id); if (j >= 0) cons[(size_t)j].push_back(i); } + auto interior = [&](const GraphNode& nd) { + if (nd.output_buffer.precision != Precision::FP32) return false; + if (nd.output_buffer.shape.size() > 8) return false; + switch (nd.op_type) { + case OpType::ADD: case OpType::ADD_CLIPPED: case OpType::SUBTRACT: + case OpType::MULTIPLY: case OpType::DIVIDE: + case OpType::SCALAR_ADD: case OpType::SCALAR_SUBTRACT: + case OpType::SCALAR_MULTIPLY: case OpType::SCALAR_DIVIDE: + case OpType::CLAMP: case OpType::GELU: case OpType::TANH: + case OpType::SILU: case OpType::RELU: + case OpType::VIEW: case OpType::RESHAPE: case OpType::FLATTEN: + case OpType::TRANSPOSE: case OpType::CAT: + return std::fabs(nd.params.scalar) < 60000.0f && std::fabs(nd.params.scale) < 60000.0f; + default: return false; + } + }; + std::vector cand(n, 0); + for (size_t i = 0; i < n; ++i) { + const GraphNode& nd = *nodes_[i]; + if (nd.op_type == OpType::PRECISION_CAST && nd.output_buffer.precision == Precision::FP32 + && !nd.input_ids.empty()) { + long j = idxof(nd.input_ids[0]); + if (j >= 0 && nodes_[(size_t)j]->output_buffer.precision == Precision::FP16) cand[i] = 1; + continue; + } + if (!interior(nd) || nd.input_ids.empty()) continue; + bool all = true; + for (size_t id : nd.input_ids) { + long j = idxof(id); + if (j < 0 || !cand[(size_t)j]) { all = false; break; } + } + cand[i] = all ? 1 : 0; + } + std::vector ok = cand; + for (size_t ri = n; ri-- > 0;) { + if (!ok[ri]) continue; + const GraphNode& nd = *nodes_[ri]; + if (retained_output_node_ids_.count(nd.id) || persistent_node_ids_.count(nd.id)) { ok[ri] = 0; continue; } + for (size_t c : cons[ri]) { + const GraphNode& cn = *nodes_[c]; + bool absorbs = (cn.op_type == OpType::PRECISION_CAST + && cn.output_buffer.precision == Precision::FP16) + || (cand[c] && ok[c]); + if (!absorbs) { ok[ri] = 0; break; } + } + } + for (size_t i = 0; i < n; ++i) { + if (!ok[i]) continue; + gpu_retype_plan_[i] = nodes_[i]->op_type == OpType::PRECISION_CAST ? 1 : 2; + } + for (size_t i = 0; i < n; ++i) { + const GraphNode& nd = *nodes_[i]; + if (nd.op_type != OpType::PRECISION_CAST || nd.output_buffer.precision != Precision::FP16 + || nd.input_ids.empty()) continue; + long j = idxof(nd.input_ids[0]); + if (j >= 0 && gpu_retype_plan_[(size_t)j] != 0) gpu_retype_plan_[i] = 1; + } +} + void CactusGraph::execute(const std::string& profile_file) { cactus_graph_mark_unadjusted(); BufferPool& pool = buffer_pool_; @@ -823,7 +991,16 @@ void CactusGraph::execute(const std::string& profile_file) { }; const bool gpu_mode = cactus_backend_metal(); + if (gpu_mode && !need_debug && !gpu_retype_built_) build_gpu_retype_plan(); + const uint8_t* rplan = (gpu_mode && !need_debug && !gpu_retype_disabled_ + && gpu_retype_plan_.size() == n) ? gpu_retype_plan_.data() : nullptr; + auto plan_of = [&](const GraphNode& node) -> uint8_t { + if (!rplan) return 0; + auto it = node_index_map_.find(node.id); + return it == node_index_map_.end() ? 0 : rplan[it->second]; + }; auto aliases_input = [&](const GraphNode& node) -> bool { + if (plan_of(node) == 1) return true; if (node.op_type == OpType::SLICE && !node.input_ids.empty()) { auto it = node_index_map_.find(node.input_ids[0]); if (it != node_index_map_.end()) { @@ -888,6 +1065,15 @@ void CactusGraph::execute(const std::string& profile_file) { GraphNode& nd = *nodes_[idx]; if (aliases_input(nd)) nd.output_buffer.external_data = nullptr; }; + size_t since_flush = 0; + std::vector gpu_live(n, 0); + auto input_gpu_live = [&](const GraphNode& nd) -> bool { + for (size_t id : nd.input_ids) { + auto it = node_index_map_.find(id); + if (it != node_index_map_.end() && gpu_live[it->second]) return true; + } + return false; + }; for (size_t i = 0; i < n; ++i) { auto& node = nodes_[i]; const OpType ot = node->op_type; @@ -900,7 +1086,13 @@ void CactusGraph::execute(const std::string& profile_file) { continue; } if (aliases_input(*node)) { - dispatch_node(*node, nodes_, node_index_map_); + if (rplan && rplan[i] == 1) { + const auto& src = get_input(*node, 0, nodes_, node_index_map_); + node->output_buffer.set_external(const_cast(static_cast(src.get_data()))); + } else { + dispatch_node(*node, nodes_, node_index_map_); + } + gpu_live[i] = input_gpu_live(*node) ? 1 : 0; for (size_t r : release_after[i]) gpu_release(r); continue; } @@ -918,13 +1110,39 @@ void CactusGraph::execute(const std::string& profile_file) { if (p) { node->output_buffer.release_to_pool(pool); node->output_buffer.set_external(p); } else node->output_buffer.resize_from_pool(pool); } + std::vector> flipped; + if (rplan && rplan[i] == 2) { + auto flip = [&](BufferDesc& b) { + if (b.precision == Precision::FP32) { flipped.push_back({&b, b.precision}); b.precision = Precision::FP16; } + }; + flip(node->output_buffer); + for (size_t id : node->input_ids) { + auto it = node_index_map_.find(id); + if (it != node_index_map_.end()) flip(nodes_[it->second]->output_buffer); + } + } bool encoded = try_encode_gpu(*node, nodes_, node_index_map_); - if (!encoded) { + for (auto& fp : flipped) fp.first->precision = fp.second; + if (!encoded && rplan && rplan[i] == 2) { + gpu_retype_disabled_ = true; cactus_metal_session_sync(); + cactus_metal_set_active(false); + cactus_metal_session_end(); + execute(profile_file); + return; + } + if (!encoded) { + if (input_gpu_live(*node)) { + cactus_metal_session_sync(); + std::fill(gpu_live.begin(), gpu_live.end(), 0); + } dispatch_node(*node, nodes_, node_index_map_); + } else { + gpu_live[i] = 1; } if (ot == OpType::PERSISTENT) populated_node_ids_.insert(node->id); for (size_t r : release_after[i]) gpu_release(r); + if (encoded && ++since_flush >= 256) { cactus_metal_session_flush(); since_flush = 0; } } cactus_metal_set_active(false); cactus_metal_session_end(); diff --git a/cactus-kernels/CMakeLists.txt b/cactus-kernels/CMakeLists.txt index b7891f2e9..20d76dc4f 100644 --- a/cactus-kernels/CMakeLists.txt +++ b/cactus-kernels/CMakeLists.txt @@ -77,7 +77,7 @@ if(APPLE) target_sources(cactus_kernels PRIVATE src/metal_backend.mm ${_msl_out}) target_include_directories(cactus_kernels PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/generated) set_source_files_properties(src/metal_backend.mm PROPERTIES COMPILE_FLAGS "-fobjc-arc") - target_link_libraries(cactus_kernels PUBLIC "-framework Metal" "-framework Foundation") + target_link_libraries(cactus_kernels PUBLIC "-framework Metal" "-framework Foundation" "-framework MetalPerformanceShaders") else() target_sources(cactus_kernels PRIVATE src/metal_backend_stub.cpp) endif() diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 68f508e4e..78a3c9cba 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -100,6 +100,22 @@ bool cactus_metal_encode_attention_i8_prefill( float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes, uint32_t sink, uint32_t ring); +bool cactus_metal_encode_binary_f32(int op, void* out, const void* a, const void* b, size_t n); +bool cactus_metal_encode_scalar_f32(int op, void* out, const void* in, size_t n, float p); +bool cactus_metal_encode_unary_f32(int op, void* out, const void* in, size_t n); +bool cactus_metal_encode_clamp(void* out, const void* in, size_t n, float lo, float hi, int f32); +bool cactus_metal_encode_glu(void* out, const void* in, size_t split, size_t inner, size_t n_out); +bool cactus_metal_encode_layer_norm(void* out, const void* in, const void* w, const void* b, + size_t rows, size_t dim, float eps); +bool cactus_metal_encode_softmax_rows(void* out, const void* in, size_t rows, size_t cols); +bool cactus_metal_encode_conv1d_k3(void* out, const void* x, const void* w, int w_int8, + const void* w_scales, uint32_t w_gs, + uint32_t Cin, uint32_t L, uint32_t Cout, uint32_t Lout, uint32_t stride); +bool cactus_metal_encode_gemm_f16(void* out, const void* lhs, const void* rhs, + uint32_t M, uint32_t K, uint32_t N, int pretransposed); +bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, const void* v, const void* mask, + uint32_t B, uint32_t T, uint32_t S, uint32_t HQ, uint32_t HKV, uint32_t D, uint32_t DV, + float scale, uint32_t causal, uint32_t pos_off, uint32_t window, float logit_cap, uint32_t mask_mode); bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* oshape, const uint32_t* sstride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes); diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index a27dd6907..fb40d22f6 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -1185,6 +1185,330 @@ kernel void strided_scatter_f16(device const half* in [[buffer(0)]], device half out[dst]=in[i]; } +kernel void binary_f32(device const float* a [[buffer(0)]], device const float* b [[buffer(1)]], + device float* y [[buffer(2)]], constant uint& n [[buffer(3)]], + constant int& op [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i>=n) return; float av=a[i], bv=b[i], r; + switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; default: r=av+bv; } + if (op==1) r=clamp(r,-65500.0f,65500.0f); + y[i]=r; +} + +kernel void scalar_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], + constant float& p [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i>=n) return; float v=in[i], r; + switch(op){ case 0: r=v+p; break; case 1: r=v-p; break; case 3: r=v/p; break; default: r=v*p; } + y[i]=r; +} + +kernel void unary_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], + uint i [[thread_position_in_grid]]) { + if (i>=n) return; float x=in[i], r; + if (op==0) r=gelu_tanh(x); + else if (op==1) r=precise::tanh(x); + else if (op==2) r=x/(1.0f+precise::exp(-x)); + else r=max(x,0.0f); + y[i]=r; +} + +kernel void clamp_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant float& lo [[buffer(3)]], + constant float& hi [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i=n) return; + uint per_outer = split*inner; + uint o = i/per_outer, r = i%per_outer; + size_t base = (size_t)o*2u*per_outer; + float a = (float)x[base + r]; + float g = (float)x[base + per_outer + r]; + y[i] = (half)(a / (1.0f + precise::exp(-g))); +} + +kernel void layer_norm_f16(device const half* in [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* b [[buffer(2)]], device half* y [[buffer(3)]], + constant uint& dim [[buffer(4)]], constant float& eps [[buffer(5)]], + constant uint& has_bias [[buffer(6)]], + uint row [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], + uint nt [[threads_per_threadgroup]], threadgroup float* red [[threadgroup(0)]]) { + device const half* x = in + (size_t)row*dim; + device half* o = y + (size_t)row*dim; + float s=0, ss=0; + for (uint i=t;i0; st>>=1){ if (t0; s>>=1){ if (t0; s>>=1){ if (t0.0f ? 1.0f/red[0] : 0.0f; + for (uint i=t;i=Lout) return; + int center = (int)(gid.x*stride); + float acc=0; + device const half* wr = w + (size_t)gid.y*Cin*3u; + for (uint c=0;c0) ? (float)xc[center-1] : 0.0f; + float x1 = (float)xc[center]; + float x2 = (center+1<(int)L) ? (float)xc[center+1] : 0.0f; + acc = fma(x0,(float)wr[c*3u],acc); + acc = fma(x1,(float)wr[c*3u+1u],acc); + acc = fma(x2,(float)wr[c*3u+2u],acc); + } + y[(size_t)gid.y*Lout + gid.x] = (half)acc; +} + +struct AttnFU { uint T, S, HQ, HKV, D, DV; float scale; uint causal, pos_off, window; float logit_cap; uint mask_mode; }; + +static inline float attn_apply_mask(float score, device const half* mask, constant AttnFU& U, + uint b, uint h, uint t, uint i) { + if (U.mask_mode != 0u) { + size_t mi = (U.mask_mode>=3u) ? (((size_t)b*U.HQ + h)*U.T + t)*U.S + i + : ((size_t)b*U.T + t)*U.S + i; + float mv = (float)mask[mi]; + if (U.mask_mode==1u || U.mask_mode==3u) { score = isfinite(mv) ? score+mv : -INFINITY; } + else if (mv == 0.0f) score = -INFINITY; + } + if (U.logit_cap > 0.0f && score > -INFINITY) score = U.logit_cap * precise::tanh(score / U.logit_cap); + return score; +} + +kernel void attn_f16(device const half* q [[buffer(0)]], device const half* k [[buffer(1)]], + device const half* v [[buffer(2)]], device half* o [[buffer(3)]], + device const half* mask [[buffer(4)]], constant AttnFU& U [[buffer(5)]], + uint3 tg [[threadgroup_position_in_grid]], uint l [[thread_index_in_threadgroup]]) { + const uint QR = 4u; + uint t0 = tg.x * QR, h = tg.y, b = tg.z; + uint kvh = h / (U.HQ / U.HKV); + device const half* kb = k + (size_t)b*U.S*U.HKV*U.D + (size_t)kvh*U.D; + device const half* vb = v + (size_t)b*U.S*U.HKV*U.DV + (size_t)kvh*U.DV; + uint d4 = U.D/4u, dv4 = U.DV/4u; + threadgroup half4 qs[4u*32u]; + uint nq = min(QR, U.T - t0); + for (uint r=0;r0u) { uint aq0 = U.pos_off + t0; kstart = (aq0>U.window) ? aq0-U.window : 0u; } + size_t kstride = (size_t)U.HKV*U.D, vstride = (size_t)U.HKV*U.DV; + for (uint kv0=kstart; kv01u){ float4 q1 = float4(qs[32u+d]); + p.y = fma(kv4.x,q1.x,p.y); p.y = fma(kv4.y,q1.y,p.y); p.y = fma(kv4.z,q1.z,p.y); p.y = fma(kv4.w,q1.w,p.y); } + if (nq>2u){ float4 q2 = float4(qs[64u+d]); + p.z = fma(kv4.x,q2.x,p.z); p.z = fma(kv4.y,q2.y,p.z); p.z = fma(kv4.z,q2.z,p.z); p.z = fma(kv4.w,q2.w,p.z); } + if (nq>3u){ float4 q3 = float4(qs[96u+d]); + p.w = fma(kv4.x,q3.x,p.w); p.w = fma(kv4.y,q3.y,p.w); p.w = fma(kv4.z,q3.z,p.w); p.w = fma(kv4.w,q3.w,p.w); } + } + sc = p * U.scale; + for (uint r=0;r<4u;++r) { + if (r >= nq) { sc[r] = -INFINITY; continue; } + uint t = t0 + r, aq = U.pos_off + t; + if (U.causal && i > aq) { sc[r] = -INFINITY; continue; } + if (U.window>0u && i < aq && (aq - i) > U.window) { sc[r] = -INFINITY; continue; } + sc[r] = attn_apply_mask(sc[r], mask, U, b, h, t, i); + } + } + float4 wgt = float4(0.0f); + bool anyrow = false; + for (uint r=0;r<4u;++r) { + float bm = simd_max(sc[r]); + if (bm > -INFINITY) { + anyrow = true; + if (bm > m[r]) { + float corr = exp(m[r] - bm); + sum[r] *= corr; acc[r][0] *= corr; acc[r][1] *= corr; + m[r] = bm; + } + wgt[r] = (sc[r] > -INFINITY) ? exp(sc[r] - m[r]) : 0.0f; + sum[r] += simd_sum(wgt[r]); + } + } + if (!anyrow) continue; + uint nk = min(32u, kend_all - kv0); + uint half_id = l >> 4u, dl = l & 15u; + for (uint j2=0; j20.0f ? 1.0f/sum[r] : 0.0f; + device half4* orow = (device half4*)(o + ((size_t)b*U.T + t0 + r)*U.HQ*U.DV + (size_t)h*U.DV); + uint ci = 0u; + for (uint dd=dl; dd0u) { uint aq0 = U.pos_off + t0; kstart = (aq0>U.window) ? aq0-U.window : 0u; } + size_t kstride = (size_t)U.HKV*U.D, vstride = (size_t)U.HKV*U.DV; + for (uint kv0=kstart; kv0= nq) { sc[r] = -INFINITY; continue; } + sc[r] = p[r] * U.scale; + uint t = t0 + r, aq = U.pos_off + t; + if (U.causal && i > aq) { sc[r] = -INFINITY; continue; } + if (U.window>0u && i < aq && (aq - i) > U.window) { sc[r] = -INFINITY; continue; } + sc[r] = attn_apply_mask(sc[r], mask, U, b, h, t, i); + } + } + float wgt[8]; + bool anyrow = false; + for (uint r=0;r<8u;++r) { + float bm = simd_max(sc[r]); + if (bm > -INFINITY) { + anyrow = true; + if (bm > m[r]) { + float corr = exp(m[r] - bm); + sum[r] *= corr; acc[r] *= corr; + m[r] = bm; + } + wgt[r] = (sc[r] > -INFINITY) ? exp(sc[r] - m[r]) : 0.0f; + sum[r] += simd_sum(wgt[r]); + } else wgt[r] = 0.0f; + } + if (!anyrow) continue; + uint nk = min(32u, kend_all - kv0); + uint half_id = l >> 4u, dl = l & 15u; + for (uint j2=0; j20.0f ? 1.0f/sum[r] : 0.0f; + device half4* orow = (device half4*)(o + ((size_t)b*U.T + t0 + r)*U.HQ*U.DV + (size_t)h*U.DV); + float4 vv; + vv[0] = acc[r][0] + simd_shuffle_xor(acc[r][0], (ushort)16); + vv[1] = acc[r][1] + simd_shuffle_xor(acc[r][1], (ushort)16); + vv[2] = acc[r][2] + simd_shuffle_xor(acc[r][2], (ushort)16); + vv[3] = acc[r][3] + simd_shuffle_xor(acc[r][3], (ushort)16); + if (l < 16u && dl < dv4) orow[dl] = half4(vv*inv); + } +} + kernel void bcast_binary_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], device half* out [[buffer(2)]], constant uint* oshape [[buffer(3)]], constant uint* astride [[buffer(4)]], constant uint* bstride [[buffer(5)]], diff --git a/cactus-kernels/src/dsp.cpp b/cactus-kernels/src/dsp.cpp index 13d3fc6fe..bee00e0af 100644 --- a/cactus-kernels/src/dsp.cpp +++ b/cactus-kernels/src/dsp.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -150,7 +152,15 @@ void cactus_rfft_f32_1d(const float* input, float* output, size_t n, const char* log2n++; } - FFTSetup fft_setup = vDSP_create_fftsetup(log2n, FFT_RADIX2); + static std::mutex fft_mu; + static std::unordered_map fft_cache; + FFTSetup fft_setup; + { + std::lock_guard lk(fft_mu); + auto it = fft_cache.find(log2n); + if (it == fft_cache.end()) it = fft_cache.emplace(log2n, vDSP_create_fftsetup(log2n, FFT_RADIX2)).first; + fft_setup = it->second; + } if (fft_setup) { std::vector buffer(padded_n, 0.0f); std::copy(input, input + n, buffer.begin()); @@ -182,7 +192,6 @@ void cactus_rfft_f32_1d(const float* input, float* output, size_t n, const char* output[i * 2 + 1] = split.imagp[i] * scale; } - vDSP_destroy_fftsetup(fft_setup); return; } } diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index e05337818..65fd53a38 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -6,6 +6,7 @@ #import #import +#import #include #include #include @@ -45,6 +46,9 @@ id psoArgmax=nil; id psoArgmaxP=nil, psoArgmaxF=nil, psoSoftcap=nil, psoAdjust=nil; id psoGather=nil; + id psoBinF32=nil, psoScaF32=nil, psoUnaF32=nil; + id psoClampF16=nil, psoClampF32=nil, psoGlu=nil; + id psoLayerNorm=nil, psoSoftmaxR=nil, psoConv1dK3=nil, psoAttnDense=nil, psoAttnD64=nil; id dummy=nil; bool ok = false; @@ -86,6 +90,10 @@ psoArgmax=pso("argmax_logits"); psoArgmaxP=pso("argmax_part"); psoArgmaxF=pso("argmax_final"); psoSoftcap=pso("softcap_f16"); psoAdjust=pso("adjust_logits"); + psoBinF32=pso("binary_f32"); psoScaF32=pso("scalar_f32"); psoUnaF32=pso("unary_f32"); + psoClampF16=pso("clamp_f16"); psoClampF32=pso("clamp_f32"); psoGlu=pso("glu_f16"); + psoLayerNorm=pso("layer_norm_f16"); psoSoftmaxR=pso("softmax_rows_f16"); + psoConv1dK3=pso("conv1d_k3_f16"); psoAttnDense=pso("attn_f16"); psoAttnD64=pso("attn_f16_d64"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; @@ -222,7 +230,7 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, id ensureEncoder() { if (!g_enc) { - g_cmd = [ctx().queue commandBuffer]; + if (!g_cmd) g_cmd = [ctx().queue commandBuffer]; g_enc = [g_cmd computeCommandEncoder]; } return g_enc; @@ -240,6 +248,8 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, std::map> g_wrapped; std::unordered_map> g_readonly; +std::unordered_map> g_dequant_cache; +std::map, MPSMatrix*> g_mps_mats; std::pair, size_t> wrapHostPtr(const void* p, size_t bytes) { uintptr_t a = (uintptr_t)p, base = a & ~(uintptr_t)16383u; size_t need = (a - base) + (bytes ? bytes : 1); @@ -290,10 +300,10 @@ inline void setBufAt(const void* p, size_t bytes, int idx) { void cactus_metal_session_begin() {} void cactus_metal_session_sync() { - if (g_enc) { - [g_enc endEncoding]; + if (g_enc) { [g_enc endEncoding]; g_enc = nil; } + if (g_cmd) { [g_cmd commit]; [g_cmd waitUntilCompleted]; - g_enc = nil; g_cmd = nil; + g_cmd = nil; } for (id b : g_pending) g_free[(size_t)b.length].push_back(b); g_pending.clear(); @@ -307,13 +317,13 @@ void cactus_metal_invalidate_host_wraps() { g_readonly.clear(); g_resident.clear(); g_resident_emb.clear(); + g_dequant_cache.clear(); + g_mps_mats.clear(); } void cactus_metal_session_flush() { - if (!g_enc) return; - [g_enc endEncoding]; - [g_cmd commit]; - g_enc = nil; g_cmd = nil; + if (g_enc) { [g_enc endEncoding]; g_enc = nil; } + if (g_cmd) { [g_cmd commit]; g_cmd = nil; } } @@ -1158,6 +1168,209 @@ bool cactus_metal_encode_attention_i8_prefill( return true; } +bool cactus_metal_encode_binary_f32(int op, void* out, const void* a, const void* b, size_t n) { + if (!ctx().ok || !ctx().psoBinF32) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; + [g_enc setComputePipelineState:ctx().psoBinF32]; + setBufAt(a, n*4, 0); setBufAt(b, n*4, 1); setBufAt(out, n*4, 2); + [g_enc setBytes:&nn length:4 atIndex:3]; [g_enc setBytes:&o length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_scalar_f32(int op, void* out, const void* in, size_t n, float p) { + if (!ctx().ok || !ctx().psoScaF32) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; + [g_enc setComputePipelineState:ctx().psoScaF32]; + setBufAt(in, n*4, 0); setBufAt(out, n*4, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; [g_enc setBytes:&p length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_unary_f32(int op, void* out, const void* in, size_t n) { + if (!ctx().ok || !ctx().psoUnaF32) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; + [g_enc setComputePipelineState:ctx().psoUnaF32]; + setBufAt(in, n*4, 0); setBufAt(out, n*4, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_clamp(void* out, const void* in, size_t n, float lo, float hi, int f32) { + id pso = f32 ? ctx().psoClampF32 : ctx().psoClampF16; + if (!ctx().ok || !pso) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; size_t es = f32?4:2; + [g_enc setComputePipelineState:pso]; + setBufAt(in, n*es, 0); setBufAt(out, n*es, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&lo length:4 atIndex:3]; [g_enc setBytes:&hi length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_glu(void* out, const void* in, size_t split, size_t inner, size_t n_out) { + if (!ctx().ok || !ctx().psoGlu) return false; + ensureEncoder(); + uint32_t sp=(uint32_t)split, in_=(uint32_t)inner, nn=(uint32_t)n_out; + [g_enc setComputePipelineState:ctx().psoGlu]; + setBufAt(in, n_out*4, 0); setBufAt(out, n_out*2, 1); + [g_enc setBytes:&sp length:4 atIndex:2]; [g_enc setBytes:&in_ length:4 atIndex:3]; [g_enc setBytes:&nn length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n_out,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_layer_norm(void* out, const void* in, const void* w, const void* b, + size_t rows, size_t dim, float eps) { + if (!ctx().ok || !ctx().psoLayerNorm) return false; + ensureEncoder(); + uint32_t d=(uint32_t)dim, hb = b ? 1u : 0u; + uint32_t T = dim >= 1024 ? 1024u : 256u; + [g_enc setComputePipelineState:ctx().psoLayerNorm]; + setBufAt(in, rows*dim*2, 0); setBufAt(w, dim*2, 1); + if (b) setBufAt(b, dim*2, 2); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + setBufAt(out, rows*dim*2, 3); + [g_enc setBytes:&d length:4 atIndex:4]; [g_enc setBytes:&eps length:4 atIndex:5]; [g_enc setBytes:&hb length:4 atIndex:6]; + [g_enc setThreadgroupMemoryLength:2*T*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + return true; +} +bool cactus_metal_encode_softmax_rows(void* out, const void* in, size_t rows, size_t cols) { + if (!ctx().ok || !ctx().psoSoftmaxR) return false; + ensureEncoder(); + uint32_t c=(uint32_t)cols; + uint32_t T = cols >= 1024 ? 1024u : 256u; + [g_enc setComputePipelineState:ctx().psoSoftmaxR]; + setBufAt(in, rows*cols*2, 0); setBufAt(out, rows*cols*2, 1); + [g_enc setBytes:&c length:4 atIndex:2]; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + return true; +} +bool cactus_metal_encode_conv1d_k3(void* out, const void* x, const void* w, int w_int8, + const void* w_scales, uint32_t w_gs, + uint32_t Cin, uint32_t L, uint32_t Cout, uint32_t Lout, uint32_t stride) { + if (!ctx().ok || !ctx().psoConv1dK3) return false; + const void* wf16 = w; + if (w_int8) { + std::lock_guard lk(g_resident_mu); + auto it = g_dequant_cache.find(w); + if (it == g_dequant_cache.end()) { + size_t rows = Cout, cols = (size_t)Cin*3; + id buf = [ctx().dev newBufferWithLength:rows*cols*2 options:MTLResourceStorageModeShared]; + if (!buf) return false; + __fp16* dst = (__fp16*)[buf contents]; + const int8_t* src = (const int8_t*)w; + if (w_gs == 0 || !w_scales) { + for (size_t i = 0; i < rows*cols; ++i) dst[i] = (__fp16)src[i]; + } else { + const __fp16* scales = (const __fp16*)w_scales; + size_t ngr = cols / w_gs; + for (size_t r = 0; r < rows; ++r) + for (size_t c = 0; c < cols; ++c) + dst[r*cols+c] = (__fp16)((float)src[r*cols+c] * (float)scales[r*ngr + c/w_gs]); + } + it = g_dequant_cache.emplace(w, buf).first; + } + wf16 = [it->second contents]; + } + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoConv1dK3]; + setBufAt(x, (size_t)Cin*L*2, 0); setBufAt(wf16, (size_t)Cout*Cin*3*2, 1); setBufAt(out, (size_t)Cout*Lout*2, 2); + [g_enc setBytes:&Cin length:4 atIndex:3]; [g_enc setBytes:&L length:4 atIndex:4]; + [g_enc setBytes:&Lout length:4 atIndex:5]; [g_enc setBytes:&stride length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(Lout,Cout,1) threadsPerThreadgroup:MTLSizeMake(64,4,1)]; + return true; +} +static MPSMatrixMultiplication* mps_gemm(uint32_t M, uint32_t N, uint32_t K, bool tr) { + static std::map, MPSMatrixMultiplication*> cache; + auto key = std::make_tuple(M,N,K,tr); + auto it = cache.find(key); + if (it != cache.end()) return it->second; + MPSMatrixMultiplication* mm = [[MPSMatrixMultiplication alloc] + initWithDevice:ctx().dev transposeLeft:false transposeRight:tr + resultRows:M resultColumns:N interiorColumns:K alpha:1.0 beta:0.0]; + cache[key] = mm; + return mm; +} + +bool cactus_metal_encode_gemm_f16(void* out, const void* lhs, const void* rhs, + uint32_t M, uint32_t K, uint32_t N, int pretransposed) { + if (!ctx().ok) return false; + static const bool mps_ok = MPSSupportsMTLDevice(ctx().dev); + auto a = bufForPtrOff(lhs, (size_t)M*K*2); + auto b = bufForPtrOff(rhs, (size_t)N*K*2); + auto c = bufForPtrOff(out, (size_t)M*N*2); + if (mps_ok && (a.second % 16)==0 && (b.second % 16)==0 && (c.second % 16)==0 + && (K % 8)==0 && (N % 8)==0) { + if (g_enc) { [g_enc endEncoding]; g_enc = nil; } + if (!g_cmd) g_cmd = [ctx().queue commandBuffer]; + MPSMatrixDescriptor* da = [MPSMatrixDescriptor matrixDescriptorWithRows:M columns:K + rowBytes:(size_t)K*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* db = pretransposed + ? [MPSMatrixDescriptor matrixDescriptorWithRows:N columns:K rowBytes:(size_t)K*2 dataType:MPSDataTypeFloat16] + : [MPSMatrixDescriptor matrixDescriptorWithRows:K columns:N rowBytes:(size_t)N*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* dc = [MPSMatrixDescriptor matrixDescriptorWithRows:M columns:N + rowBytes:(size_t)N*2 dataType:MPSDataTypeFloat16]; + auto mat = [](id buf, size_t off, MPSMatrixDescriptor* d) -> MPSMatrix* { + auto key = std::make_tuple((__bridge void*)buf, off, (size_t)d.rows, (size_t)d.columns); + auto it = g_mps_mats.find(key); + if (it != g_mps_mats.end()) return it->second; + if (g_mps_mats.size() > 4096) g_mps_mats.clear(); + MPSMatrix* m = [[MPSMatrix alloc] initWithBuffer:buf offset:off descriptor:d]; + g_mps_mats[key] = m; + return m; + }; + MPSMatrix* ma = mat(a.first, a.second, da); + MPSMatrix* mb = mat(b.first, b.second, db); + MPSMatrix* mc = mat(c.first, c.second, dc); + [mps_gemm(M, N, K, pretransposed != 0) encodeToCommandBuffer:g_cmd + leftMatrix:ma rightMatrix:mb resultMatrix:mc]; + return true; + } + if (!ctx().psoGdense || !ctx().psoStrided) return false; + ensureEncoder(); + id tbuf = nil; + if (!pretransposed) { + tbuf = recycled((size_t)K*N*2); + if (!tbuf) return false; + uint32_t oshape[2] = { N, K }, sstride[2] = { 1u, N }; + uint32_t total = N*K; + [g_enc setComputePipelineState:ctx().psoStrided]; + setBufAt(rhs, (size_t)K*N*2, 0); [g_enc setBuffer:tbuf offset:0 atIndex:1]; + [g_enc setBytes:oshape length:8 atIndex:2]; [g_enc setBytes:sstride length:8 atIndex:3]; + uint32_t nd=2, base=0; + [g_enc setBytes:&nd length:4 atIndex:4]; [g_enc setBytes:&total length:4 atIndex:5]; [g_enc setBytes:&base length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } + [g_enc setComputePipelineState:ctx().psoGdense]; + setBufAt(lhs, (size_t)M*K*2, 0); + if (tbuf) [g_enc setBuffer:tbuf offset:0 atIndex:1]; else setBufAt(rhs, (size_t)N*K*2, 1); + setBufAt(out, (size_t)M*N*2, 2); + [g_enc setBytes:&K length:4 atIndex:3]; [g_enc setBytes:&N length:4 atIndex:4]; [g_enc setBytes:&M length:4 atIndex:5]; + [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + return true; +} +bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, const void* v, const void* mask, + uint32_t B, uint32_t T, uint32_t S, uint32_t HQ, uint32_t HKV, uint32_t D, uint32_t DV, + float scale, uint32_t causal, uint32_t pos_off, uint32_t window, float logit_cap, uint32_t mask_mode) { + if (!ctx().ok || !ctx().psoAttnDense) return false; + if (D > 128 || DV > 128 || (D % 4) != 0 || (DV % 4) != 0 || HKV == 0 || HQ % HKV != 0) return false; + ensureEncoder(); + struct { uint32_t T,S,HQ,HKV,D,DV; float scale; uint32_t causal,pos_off,window; float logit_cap; uint32_t mask_mode; } U = + { T,S,HQ,HKV,D,DV, scale, causal,pos_off,window, logit_cap, mask_mode }; + bool d64 = (D <= 64 && DV <= 64 && ctx().psoAttnD64); + [g_enc setComputePipelineState:(d64 ? ctx().psoAttnD64 : ctx().psoAttnDense)]; + setBufAt(q, (size_t)B*T*HQ*D*2, 0); setBufAt(k, (size_t)B*S*HKV*D*2, 1); + setBufAt(v, (size_t)B*S*HKV*DV*2, 2); setBufAt(out, (size_t)B*T*HQ*DV*2, 3); + if (mask_mode && mask) { + size_t mb = (mask_mode>=3u) ? (size_t)B*HQ*T*S*2 : (size_t)B*T*S*2; + setBufAt(mask, mb, 4); + } else [g_enc setBuffer:ctx().dummy offset:0 atIndex:4]; + [g_enc setBytes:&U length:sizeof(U) atIndex:5]; + [g_enc dispatchThreadgroups:MTLSizeMake(d64 ? (T+7)/8 : (T+3)/4,HQ,B) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + return true; +} + bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* oshape, const uint32_t* sstride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes) { if (!ctx().ok || ndim == 0 || ndim > 8) return false; diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 14fb056e6..fcd068603 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -60,6 +60,19 @@ bool cactus_metal_encode_attention_fused_i8(void*, const void*, const void*, con bool cactus_metal_encode_attention_i8_prefill(void*, const void*, const void*, const void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, size_t, size_t, size_t, size_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_binary_f32(int, void*, const void*, const void*, size_t) { return false; } +bool cactus_metal_encode_scalar_f32(int, void*, const void*, size_t, float) { return false; } +bool cactus_metal_encode_unary_f32(int, void*, const void*, size_t) { return false; } +bool cactus_metal_encode_clamp(void*, const void*, size_t, float, float, int) { return false; } +bool cactus_metal_encode_glu(void*, const void*, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_layer_norm(void*, const void*, const void*, const void*, size_t, size_t, float) { return false; } +bool cactus_metal_encode_softmax_rows(void*, const void*, size_t, size_t) { return false; } +bool cactus_metal_encode_conv1d_k3(void*, const void*, const void*, int, const void*, uint32_t, + uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_gemm_f16(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_attention_f16(void*, const void*, const void*, const void*, const void*, + uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, + float, uint32_t, uint32_t, uint32_t, float, uint32_t) { return false; } bool cactus_metal_encode_strided_copy(void*, const void*, const uint32_t*, const uint32_t*, uint32_t, uint32_t, uint32_t, size_t, size_t) { return false; } bool cactus_metal_encode_bcast_binary(int, void*, const void*, const void*, const uint32_t*, diff --git a/python/cactus/cli/compile.py b/python/cactus/cli/compile.py index 0bc7ef1b7..429cbdb39 100644 --- a/python/cactus/cli/compile.py +++ b/python/cactus/cli/compile.py @@ -100,6 +100,7 @@ def build_binary( "-framework", "CoreML", "-framework", "Foundation", "-framework", "Metal", + "-framework", "MetalPerformanceShaders", "-framework", "Security", "-framework", "SystemConfiguration", "-framework", "CFNetwork", diff --git a/python/cactus/cli/runtime.py b/python/cactus/cli/runtime.py index f9e051865..54956a7c4 100644 --- a/python/cactus/cli/runtime.py +++ b/python/cactus/cli/runtime.py @@ -78,6 +78,7 @@ def _link_python_runtime_library(*, static_lib, library_path): "-framework", "CoreML", "-framework", "Foundation", "-framework", "Metal", + "-framework", "MetalPerformanceShaders", "-framework", "Security", "-framework", "SystemConfiguration", "-framework", "CFNetwork", From c64dceab0a71254f234f9ad45f8117609e81dcc9 Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 18:34:29 +0200 Subject: [PATCH 14/36] Generalize gemma decode into cactus-graph with automatic Metal fusion Replaces the hardcoded gemma-4-E2B execute_gpu_fused path with a general fusion-plan compiler in the graph executor: per-shape plans pattern-match the loaded graph once and fuse decomposed rope + q/k/v norms + KV append + cached attention into the fused attention kernel (owner and KV-sharing variants), sibling quantized gemvs into transform_batch/gemv_cat with consumer-ordering guards, residual rms-norm chains with folded layer scales into rms_norm_add_rms, gelu-swiglu into swiglu_transform+gemv, PLE pairs into transform_gemv, and the orthogonal lm_head with softcap into the GPU argmax/sampling tail. Plans add a liveness-scanned activation arena, a compact execution list, an embedding-fold prologue, and encode-failure rollback to the generic path. Decode matches the removed hand-fused path (119-121/111-114/95-100 tps vs 122/115/101) and prefill improves (L=16 205 vs 124 tps); tokens stay identical to CPU for cq4/cq3 and all models in the weights directory pass llm/stt/vlm suites on both backends. Renames graph backend identifiers from gpu to metal, adds ComputeBackend::METAL with Python binding, merges overlapping no-copy host buffer wraps, and runs batch generation at a fixed slot-count shape so every batch size executes identically. Signed-off-by: jakmro --- cactus-engine/src/model.cpp | 45 +- cactus-graph/CMakeLists.txt | 3 +- cactus-graph/cactus_graph.h | 23 +- cactus-graph/src/execute.cpp | 152 +++- cactus-graph/src/gpu_fused.cpp | 508 ------------ cactus-graph/src/metal_plan.cpp | 913 ++++++++++++++++++++++ cactus-graph/src/metal_runtime.cpp | 198 +++++ cactus-kernels/metal_backend.h | 1 + cactus-kernels/src/metal_backend.mm | 39 +- cactus-kernels/src/metal_backend_stub.cpp | 1 + python/cactus/bindings/cactus.py | 1 + 11 files changed, 1303 insertions(+), 581 deletions(-) delete mode 100644 cactus-graph/src/gpu_fused.cpp create mode 100644 cactus-graph/src/metal_plan.cpp create mode 100644 cactus-graph/src/metal_runtime.cpp diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 35c41895c..52fb22e7b 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -562,8 +562,8 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, } else if (decode_route_ != DecodeRoute::ENCODER_CROSS_KV_STEP && !components_.count("audio_encoder")) { CACTUS_LOG_WARN("model", "Bundle has no decoder_prefill_chunk component; prompts will prefill token-by-token (prefill speed ~= decode speed)."); } - if (decoder_ && decoder_->graph) decoder_->graph->prewarm_gpu_quant_weights(); - if (decoder_prefill_ && decoder_prefill_->graph) decoder_prefill_->graph->prewarm_gpu_quant_weights(); + if (decoder_ && decoder_->graph) decoder_->graph->prewarm_metal_quant_weights(); + if (decoder_prefill_ && decoder_prefill_->graph) decoder_prefill_->graph->prewarm_metal_quant_weights(); if (components_.count("decoder_embed_chunk") && components_.at("decoder_embed_chunk").graph) { decoder_embed_ = &components_.at("decoder_embed_chunk"); } @@ -900,30 +900,31 @@ void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/, b if (decode_route_ == DecodeRoute::DIRECT_DECODER_STEP) { write_int_input(*decoder_, "input_ids", static_cast(token_id)); write_int_input(*decoder_, "position_ids", static_cast(position)); - const bool fused = use_fused && cactus_backend_fused(); - if (!fused) cactus_graph_set_prefill_consistent(true); - if (!fused || !decoder_->graph->execute_gpu_fused()) decoder_->graph->execute(); + if (!use_fused || !cactus_backend_metal()) cactus_graph_set_prefill_consistent(true); + decoder_->graph->execute(); cactus_graph_set_prefill_consistent(false); maybe_capture_handoff_probe_hidden(*decoder_); return; } if (!use_fused) cactus_graph_set_prefill_consistent(true); - const bool fused = use_fused && cactus_backend_fused(); - if (fused) { + if (use_fused && cactus_backend_metal()) { if (ple_probe_state_ == 0) ple_probe_state_ = encoder_->graph->extract_ple_pathway(fused_embed_ctx_) ? 1 : 2; if (ple_probe_state_ == 1) { fused_embed_ctx_.token_id = static_cast(token_id); fused_embed_ctx_.position = static_cast(position); cactus_graph_set_fused_embed(&fused_embed_ctx_); - bool ok = decoder_->graph->execute_gpu_fused(); + write_int_input(*decoder_, "position_ids", static_cast(position)); + decoder_->graph->execute(); cactus_graph_set_fused_embed(nullptr); - if (ok) { cactus_graph_set_prefill_consistent(false); maybe_capture_handoff_probe_hidden(*decoder_); return; } + cactus_graph_set_prefill_consistent(false); + maybe_capture_handoff_probe_hidden(*decoder_); + return; } } run_encoder_step(token_id, position); copy_component_outputs_to_inputs(*encoder_, *decoder_); - if (!fused || !decoder_->graph->execute_gpu_fused()) decoder_->graph->execute(); + decoder_->graph->execute(); cactus_graph_set_prefill_consistent(false); maybe_capture_handoff_probe_hidden(*decoder_); } @@ -1020,9 +1021,13 @@ std::vector> Model::generate_batch(const std::vector 1 || cactus_backend_metal()) { + exec_batch = std::max(batch, decoder_cache_num_slots()); + } reset_component_cache_states(*decoder_); - if (cached) set_component_batch(*encoder_, batch); - set_component_batch(*decoder_, batch); + if (cached) set_component_batch(*encoder_, exec_batch); + set_component_batch(*decoder_, exec_batch); std::vector> out(batch); std::vector fed(batch, 0); @@ -1036,13 +1041,17 @@ std::vector> Model::generate_batch(const std::vector tokens(batch); - std::vector positions(batch); + std::vector tokens(exec_batch); + std::vector positions(exec_batch); while (remaining > 0) { for (size_t b = 0; b < batch; ++b) { positions[b] = fed[b]; tokens[b] = (fed[b] < prompts[b].size()) ? prompts[b][fed[b]] : last[b]; } + for (size_t b = batch; b < exec_batch; ++b) { + positions[b] = positions[0]; + tokens[b] = tokens[0]; + } run_step_batch(tokens, positions); std::vector sampled = argmax_component_logits_batch(*decoder_, batch); for (size_t b = 0; b < batch; ++b) { @@ -1959,7 +1968,7 @@ uint32_t Model::argmax_logits_at(const BufferDesc& desc, void* ptr, size_t row_o { uint32_t gidx; float gbest, gsecond; if (!tool_dense && tool_bias.empty() && vocab_bias_.empty() && suppressed_token_id_ < 0 && - cactus_graph_gpu_argmax(&gidx, &gbest, &gsecond)) { + cactus_graph_metal_argmax(&gidx, &gbest, &gsecond)) { if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(gbest, gsecond); return gidx; } @@ -2094,7 +2103,7 @@ uint32_t Model::sample_component_logits(Component& comp, float temperature, floa auto get = [&](size_t i) -> float { return fp16 ? (float)h[i] : f[i]; }; auto put = [&](size_t i, float v) { if (fp16) h[i] = (__fp16)v; else f[i] = v; }; - if (samp_ctx_active_ && !cactus_graph_gpu_adjusted()) { + if (samp_ctx_active_ && !cactus_graph_metal_adjusted()) { if (samp_penalty_ != 1.0f) { for (uint32_t id : samp_recent_) { if (id >= vocab) continue; @@ -2107,8 +2116,8 @@ uint32_t Model::sample_component_logits(Component& comp, float temperature, floa } if (greedy) { uint32_t gidx; float gbest, gsecond; - if ((!samp_has_bias_ || cactus_graph_gpu_argmax_biased()) && - cactus_graph_gpu_argmax(&gidx, &gbest, &gsecond)) { + if ((!samp_has_bias_ || cactus_graph_metal_argmax_biased()) && + cactus_graph_metal_argmax(&gidx, &gbest, &gsecond)) { if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(gbest, gsecond); return gidx; } diff --git a/cactus-graph/CMakeLists.txt b/cactus-graph/CMakeLists.txt index 6f2704ce8..9c431a855 100644 --- a/cactus-graph/CMakeLists.txt +++ b/cactus-graph/CMakeLists.txt @@ -12,7 +12,8 @@ add_library(cactus_graph STATIC src/core.cpp src/builder.cpp src/execute.cpp - src/gpu_fused.cpp + src/metal_runtime.cpp + src/metal_plan.cpp src/io.cpp src/param_io.cpp src/ops_math.cpp diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 3e4e2f6c7..1701cbbbd 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -19,7 +19,6 @@ int cactus_backend_select(const char* backend); bool cactus_backend_metal(); -bool cactus_backend_fused(); namespace cactus { @@ -85,7 +84,7 @@ class Logger { #define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) #define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) -enum class ComputeBackend { CPU, NPU }; +enum class ComputeBackend { CPU, NPU, METAL }; enum class Activation { SILU, GELU, GELU_ERF, RELU, SIGMOID, TANH }; @@ -466,7 +465,7 @@ struct FusedEmbedCtx { }; void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx); -bool cactus_graph_gpu_argmax(uint32_t* idx, float* best, float* second); +bool cactus_graph_metal_argmax(uint32_t* idx, float* best, float* second); void cactus_graph_mark_unadjusted(); void cactus_graph_set_prefill_consistent(bool on); bool cactus_graph_prefill_consistent(); @@ -475,8 +474,8 @@ void cactus_graph_set_sampling(const uint32_t* recent, int n_recent, float rep_p const float* bias_dense, size_t bias_len, long long suppressed); void cactus_graph_clear_sampling(); -bool cactus_graph_gpu_adjusted(); -bool cactus_graph_gpu_argmax_biased(); +bool cactus_graph_metal_adjusted(); +bool cactus_graph_metal_argmax_biased(); class CactusGraph { public: @@ -731,8 +730,8 @@ class CactusGraph { void invalidate_persistent(size_t persistent_node_id); void execute(const std::string& profile_file = ""); - bool execute_gpu_fused(); - void build_gpu_retype_plan(); + void build_metal_retype_plan(); + std::unordered_map metal_plans_; bool extract_ple_pathway(FusedEmbedCtx& ctx) const; void hard_reset(); void soft_reset(); @@ -759,7 +758,7 @@ class CactusGraph { const std::vector& backup); void allocate_buffers(); size_t get_node_count() const; - void prewarm_gpu_quant_weights(); + void prewarm_metal_quant_weights(); void set_runtime_input_shape(size_t node_id, const std::vector& shape); void set_input_dynamic_dims(size_t node_id, const std::vector& dynamic_dims); bool has_dynamic_shapes() const { return has_dynamic_shapes_; } @@ -786,10 +785,10 @@ class CactusGraph { std::unordered_set populated_node_ids_; std::unordered_set embedded_input_node_ids_; std::unordered_set retained_output_node_ids_; - std::vector gpu_retype_plan_; - bool gpu_retype_built_ = false; - bool gpu_retype_disabled_ = false; - std::unordered_map> gpu_persistent_acts_; + std::vector metal_retype_plan_; + bool metal_retype_built_ = false; + bool metal_retype_disabled_ = false; + std::unordered_map> metal_persistent_acts_; }; namespace GraphFile { diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 5ba9bdbeb..1c8680d38 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -27,7 +27,6 @@ int cactus_backend_select(const char* backend) { return -1; } bool cactus_backend_metal() { return g_backend_override != 0 && cactus_metal_available(); } -bool cactus_backend_fused() { return cactus_backend_metal(); } using ComputeFn = void(*)(GraphNode&, const nodes_vector&, const node_index_map_t&); @@ -456,7 +455,7 @@ static void bcast_strides(const std::vector& in_shape, const std::vector out[d] = (d < off) ? 0u : (in_shape[d-off]==1 ? 0u : (uint32_t)istr[d-off]); } -static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const node_index_map_t& map) { +static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const node_index_map_t& map) { BufferDesc& out = node.output_buffer; auto fp16 = [](const BufferDesc& b){ return b.precision == Precision::FP16; }; switch (node.op_type) { @@ -846,10 +845,24 @@ static bool try_encode_gpu(GraphNode& node, const nodes_vector& nodes, const nod } } -void CactusGraph::build_gpu_retype_plan() { - gpu_retype_built_ = true; +struct MetalFusePlan; +MetalFusePlan* cactus_metal_plan_build(const std::vector>& nodes, + const std::unordered_map& map); +void cactus_metal_plan_free(MetalFusePlan* p); +void cactus_graph_metal_tail_commit(); +void cactus_metal_plan_disable(MetalFusePlan* p); +bool cactus_metal_plan_fold(MetalFusePlan* p, const std::vector>& nodes); +const std::vector* cactus_metal_plan_exec_list(const MetalFusePlan* p); +void* cactus_metal_plan_arena_ptr(const MetalFusePlan* p, size_t i); +int32_t cactus_metal_plan_action(const MetalFusePlan* p, size_t i); +bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, + const std::vector>& nodes, + const std::unordered_map& map); + +void CactusGraph::build_metal_retype_plan() { + metal_retype_built_ = true; const size_t n = nodes_.size(); - gpu_retype_plan_.assign(n, 0); + metal_retype_plan_.assign(n, 0); auto idxof = [&](size_t id) -> long { auto it = node_index_map_.find(id); return it == node_index_map_.end() ? -1 : (long)it->second; @@ -905,14 +918,14 @@ void CactusGraph::build_gpu_retype_plan() { } for (size_t i = 0; i < n; ++i) { if (!ok[i]) continue; - gpu_retype_plan_[i] = nodes_[i]->op_type == OpType::PRECISION_CAST ? 1 : 2; + metal_retype_plan_[i] = nodes_[i]->op_type == OpType::PRECISION_CAST ? 1 : 2; } for (size_t i = 0; i < n; ++i) { const GraphNode& nd = *nodes_[i]; if (nd.op_type != OpType::PRECISION_CAST || nd.output_buffer.precision != Precision::FP16 || nd.input_ids.empty()) continue; long j = idxof(nd.input_ids[0]); - if (j >= 0 && gpu_retype_plan_[(size_t)j] != 0) gpu_retype_plan_[i] = 1; + if (j >= 0 && metal_retype_plan_[(size_t)j] != 0) metal_retype_plan_[i] = 1; } } @@ -990,10 +1003,25 @@ void CactusGraph::execute(const std::string& profile_file) { return true; }; - const bool gpu_mode = cactus_backend_metal(); - if (gpu_mode && !need_debug && !gpu_retype_built_) build_gpu_retype_plan(); - const uint8_t* rplan = (gpu_mode && !need_debug && !gpu_retype_disabled_ - && gpu_retype_plan_.size() == n) ? gpu_retype_plan_.data() : nullptr; + const bool metal_mode = cactus_backend_metal(); + if (metal_mode && !need_debug && !metal_retype_built_) build_metal_retype_plan(); + MetalFusePlan* fplan = nullptr; + if (metal_mode && !need_debug) { + uint64_t sig = 1469598103934665603ull; + for (auto& np : nodes_) { + if (np->op_type != OpType::INPUT) continue; + for (size_t d : np->output_buffer.shape) { + sig ^= (uint64_t)d + 0x9e3779b97f4a7c15ull; + sig *= 1099511628211ull; + } + } + auto it = metal_plans_.find(sig); + if (it == metal_plans_.end()) + it = metal_plans_.emplace(sig, cactus_metal_plan_build(nodes_, node_index_map_)).first; + fplan = it->second; + } + const uint8_t* rplan = (metal_mode && !need_debug && !metal_retype_disabled_ + && metal_retype_plan_.size() == n) ? metal_retype_plan_.data() : nullptr; auto plan_of = [&](const GraphNode& node) -> uint8_t { if (!rplan) return 0; auto it = node_index_map_.find(node.id); @@ -1014,7 +1042,7 @@ void CactusGraph::execute(const std::string& profile_file) { } } if (node.op_type == OpType::INDEX && node.params.axis == 0) return true; - if (!gpu_mode) return false; + if (!metal_mode) return false; if (node.op_type == OpType::VIEW || node.op_type == OpType::RESHAPE || node.op_type == OpType::FLATTEN) return true; if (node.op_type == OpType::PRECISION_CAST && !node.input_ids.empty()) { @@ -1058,23 +1086,27 @@ void CactusGraph::execute(const std::string& profile_file) { release_after[last_use[i]].push_back(i); } - if (gpu_mode && !need_debug) { + if (metal_mode && !need_debug) { cactus_metal_session_begin(); cactus_metal_set_active(true); - auto gpu_release = [&](size_t idx) { + auto metal_release = [&](size_t idx) { GraphNode& nd = *nodes_[idx]; if (aliases_input(nd)) nd.output_buffer.external_data = nullptr; }; size_t since_flush = 0; - std::vector gpu_live(n, 0); - auto input_gpu_live = [&](const GraphNode& nd) -> bool { + if (fplan) cactus_metal_plan_fold(fplan, nodes_); + std::vector metal_live(n, 0); + auto input_metal_live = [&](const GraphNode& nd) -> bool { for (size_t id : nd.input_ids) { auto it = node_index_map_.find(id); - if (it != node_index_map_.end() && gpu_live[it->second]) return true; + if (it != node_index_map_.end() && metal_live[it->second]) return true; } return false; }; - for (size_t i = 0; i < n; ++i) { + const std::vector* elist = fplan ? cactus_metal_plan_exec_list(fplan) : nullptr; + size_t iter_count = elist ? elist->size() : n; + for (size_t ii = 0; ii < iter_count; ++ii) { + size_t i = elist ? (size_t)(*elist)[ii] : ii; auto& node = nodes_[i]; const OpType ot = node->op_type; if (ot == OpType::INPUT) continue; @@ -1082,7 +1114,28 @@ void CactusGraph::execute(const std::string& profile_file) { || ot == OpType::RECURRENT_CACHE_STATE) { dispatch_node(*node, nodes_, node_index_map_); populated_node_ids_.insert(node->id); - for (size_t r : release_after[i]) gpu_release(r); + for (size_t r : release_after[i]) metal_release(r); + continue; + } + int32_t fact = fplan ? cactus_metal_plan_action(fplan, i) : -1; + if (fact == -2) { + for (size_t r : release_after[i]) metal_release(r); + continue; + } + if (fact == -3) { + size_t need = node->output_buffer.byte_size; + auto pit = metal_persistent_acts_.find(node->id); + void* p = nullptr; + if (pit != metal_persistent_acts_.end() && pit->second.second >= need) p = pit->second.first; + else { + if (pit != metal_persistent_acts_.end()) cactus_metal_free_shared(pit->second.first); + p = cactus_metal_alloc_pooled(need); + if (p) metal_persistent_acts_[node->id] = { p, need }; + } + if (p) { node->output_buffer.release_to_pool(pool); node->output_buffer.set_external(p); } + else node->output_buffer.resize_from_pool(pool); + metal_live[i] = 1; + for (size_t r : release_after[i]) metal_release(r); continue; } if (aliases_input(*node)) { @@ -1092,23 +1145,43 @@ void CactusGraph::execute(const std::string& profile_file) { } else { dispatch_node(*node, nodes_, node_index_map_); } - gpu_live[i] = input_gpu_live(*node) ? 1 : 0; - for (size_t r : release_after[i]) gpu_release(r); + metal_live[i] = input_metal_live(*node) ? 1 : 0; + for (size_t r : release_after[i]) metal_release(r); continue; } if (preallocates_output(*node)) { - size_t need = node->output_buffer.byte_size; - auto pit = gpu_persistent_acts_.find(node->id); - void* p = nullptr; - if (pit != gpu_persistent_acts_.end() && pit->second.second >= need) { - p = pit->second.first; + void* ap = fplan ? cactus_metal_plan_arena_ptr(fplan, i) : nullptr; + if (ap) { + node->output_buffer.release_to_pool(pool); + node->output_buffer.set_external(ap); } else { - if (pit != gpu_persistent_acts_.end()) cactus_metal_free_shared(pit->second.first); - p = cactus_metal_alloc_shared(need); - if (p) gpu_persistent_acts_[node->id] = { p, need }; + size_t need = node->output_buffer.byte_size; + auto pit = metal_persistent_acts_.find(node->id); + void* p = nullptr; + if (pit != metal_persistent_acts_.end() && pit->second.second >= need) { + p = pit->second.first; + } else { + if (pit != metal_persistent_acts_.end()) cactus_metal_free_shared(pit->second.first); + p = cactus_metal_alloc_pooled(need); + if (p) metal_persistent_acts_[node->id] = { p, need }; + } + if (p) { node->output_buffer.release_to_pool(pool); node->output_buffer.set_external(p); } + else node->output_buffer.resize_from_pool(pool); } - if (p) { node->output_buffer.release_to_pool(pool); node->output_buffer.set_external(p); } - else node->output_buffer.resize_from_pool(pool); + } + if (fact >= 0) { + if (cactus_metal_plan_encode(fplan, fact, nodes_, node_index_map_)) { + metal_live[i] = 1; + for (size_t r : release_after[i]) metal_release(r); + if (++since_flush >= 48) { cactus_metal_session_flush(); since_flush = 0; } + continue; + } + cactus_metal_plan_disable(fplan); + cactus_metal_session_sync(); + cactus_metal_set_active(false); + cactus_metal_session_end(); + execute(profile_file); + return; } std::vector> flipped; if (rplan && rplan[i] == 2) { @@ -1121,10 +1194,10 @@ void CactusGraph::execute(const std::string& profile_file) { if (it != node_index_map_.end()) flip(nodes_[it->second]->output_buffer); } } - bool encoded = try_encode_gpu(*node, nodes_, node_index_map_); + bool encoded = try_encode_metal(*node, nodes_, node_index_map_); for (auto& fp : flipped) fp.first->precision = fp.second; if (!encoded && rplan && rplan[i] == 2) { - gpu_retype_disabled_ = true; + metal_retype_disabled_ = true; cactus_metal_session_sync(); cactus_metal_set_active(false); cactus_metal_session_end(); @@ -1132,20 +1205,21 @@ void CactusGraph::execute(const std::string& profile_file) { return; } if (!encoded) { - if (input_gpu_live(*node)) { + if (input_metal_live(*node)) { cactus_metal_session_sync(); - std::fill(gpu_live.begin(), gpu_live.end(), 0); + std::fill(metal_live.begin(), metal_live.end(), 0); } dispatch_node(*node, nodes_, node_index_map_); } else { - gpu_live[i] = 1; + metal_live[i] = 1; } if (ot == OpType::PERSISTENT) populated_node_ids_.insert(node->id); - for (size_t r : release_after[i]) gpu_release(r); - if (encoded && ++since_flush >= 256) { cactus_metal_session_flush(); since_flush = 0; } + for (size_t r : release_after[i]) metal_release(r); + if (encoded && ++since_flush >= 48) { cactus_metal_session_flush(); since_flush = 0; } } cactus_metal_set_active(false); cactus_metal_session_end(); + cactus_graph_metal_tail_commit(); return; } @@ -1641,7 +1715,7 @@ void CactusGraph::soft_reset_keep_pool() { debug_nodes_.clear(); } -void CactusGraph::prewarm_gpu_quant_weights() { +void CactusGraph::prewarm_metal_quant_weights() { const bool gpu = cactus_backend_metal(); if (!gpu) return; for (auto& np : nodes_) { diff --git a/cactus-graph/src/gpu_fused.cpp b/cactus-graph/src/gpu_fused.cpp deleted file mode 100644 index 2a2848f99..000000000 --- a/cactus-graph/src/gpu_fused.cpp +++ /dev/null @@ -1,508 +0,0 @@ -#include "../cactus_graph.h" -#include "cactus_kernels.h" -#include "metal_backend.h" -#include -#include -#include -#include -#include - -namespace { - -constexpr int NL = 35, HID = 1536, NQH = 8, PLE_DIM = 256; -constexpr float RMS_EPS = 1e-6f, GATE_SCALE = 0.015625f, SOFTCAP = 30.0f; - -constexpr int FLUSH_LAYERS = 3; - -struct LayerW { - CactusQuantMatrix q, o, ple_down, ple_up, gate, up, down; - CactusQuantMatrix k, v; - const void *in_norm, *q_norm, *k_norm, *post_attn, *pre_ffn, *post_ffn, *post_ple, *scalar; - bool shared, global; - int hd, mlp; - void *kc, *vc; -}; - -struct Arena { - std::vector> bufs; - size_t cur = 0; - void reset() { cur = 0; } - void* fresh(size_t bytes) { - if (cur >= bufs.size()) { - void* p = cactus_metal_alloc_shared(bytes); - if (p) std::memset(p, 0, bytes); - bufs.push_back({p, bytes}); - } - return bufs[cur++].first; - } -}; - -struct GpuModel { - bool init = false; - const void* owner = nullptr; - LayerW L[NL]; - size_t kc_idx[15], vc_idx[15]; - const void *vnorm, *vnorm_g; - const void *cosL, *sinL, *cosG, *sinG; - const float *position; - const void *hidden; - const void *ple; - const void *final_norm; - CactusQuantMatrix lm_head; - void *cos_s, *sin_s, *cos_g, *sin_g; - void *amax = nullptr; - Arena arena; -}; -static GpuModel G; - -static bool g_gpu_argmax_valid = false; -static const float* g_gpu_argmax_buf = nullptr; - -} - -static bool g_last_step_adjusted = false; -static bool g_last_amax_biased = false; -static bool g_prefill_consistent = false; - -void cactus_graph_mark_unadjusted() { - g_last_step_adjusted = false; - g_last_amax_biased = false; - g_gpu_argmax_valid = false; -} - -void cactus_graph_set_prefill_consistent(bool on) { g_prefill_consistent = on; } -bool cactus_graph_prefill_consistent() { return g_prefill_consistent; } - -struct GSampling { - bool active = false; - float rep_penalty = 1.0f; - std::vector recent; - const float* bias_dense = nullptr; - size_t bias_len = 0; - long long suppressed = -1; -}; -static GSampling g_samp; -static bool g_step_adjusted = false; -static bool g_step_amax_biased = false; -static float* g_bias_dense = nullptr; -static size_t g_bias_dense_n = 0; - -void cactus_graph_set_sampling(const uint32_t* recent, int n_recent, float rep_penalty, - const float* bias_dense, size_t bias_len, - long long suppressed) { - g_samp.active = true; - g_samp.rep_penalty = rep_penalty; - g_samp.recent.assign(recent, recent + (n_recent > 0 ? n_recent : 0)); - g_samp.bias_dense = bias_dense; - g_samp.bias_len = bias_dense ? bias_len : 0; - g_samp.suppressed = suppressed; -} -void cactus_graph_clear_sampling() { g_samp = GSampling(); } -bool cactus_graph_gpu_adjusted() { return g_last_step_adjusted; } -bool cactus_graph_gpu_argmax_biased() { return g_last_amax_biased; } - -static const float* update_bias_dense(size_t vocab) { - if (!g_samp.bias_dense || g_samp.bias_len == 0) return nullptr; - if (!g_bias_dense || g_bias_dense_n < vocab) { - g_bias_dense = (float*)cactus_metal_alloc_shared(vocab * sizeof(float)); - if (!g_bias_dense) return nullptr; - g_bias_dense_n = vocab; - } - size_t n = g_samp.bias_len < vocab ? g_samp.bias_len : vocab; - std::memcpy(g_bias_dense, g_samp.bias_dense, n * sizeof(float)); - if (n < vocab) std::memset(g_bias_dense + n, 0, (vocab - n) * sizeof(float)); - return g_bias_dense; -} - -static inline bool sampling_needs_adjust() { - return g_samp.active && - ((g_samp.rep_penalty != 1.0f && !g_samp.recent.empty()) || g_samp.suppressed >= 0); -} - -bool cactus_graph_gpu_argmax(uint32_t* idx, float* best, float* second) { - if (!g_gpu_argmax_valid || !g_gpu_argmax_buf) return false; - g_gpu_argmax_valid = false; - *best = g_gpu_argmax_buf[0]; *second = g_gpu_argmax_buf[1]; - *idx = (uint32_t)g_gpu_argmax_buf[2]; - return true; -} - -static FusedEmbedCtx g_fe; -void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx) { - if (ctx && ctx->ok) g_fe = *ctx; else g_fe.ok = false; -} - -bool CactusGraph::extract_ple_pathway(FusedEmbedCtx& ctx) const { - - if (nodes_.size() != 23) return false; - auto op = [&](size_t i){ return nodes_[i]->op_type; }; - if (op(3) != OpType::INPUT || op(4) != OpType::INPUT || op(5) != OpType::INPUT) return false; - if (op(8) != OpType::EMBEDDING || op(12) != OpType::MATMUL || op(19) != OpType::RMS_NORM) return false; - if (op(7) != OpType::SCALAR_MULTIPLY || op(9) != OpType::SCALAR_MULTIPLY || - op(14) != OpType::SCALAR_MULTIPLY || op(22) != OpType::SCALAR_MULTIPLY) return false; - const BufferDesc& pleW = nodes_[3]->output_buffer; - const BufferDesc& projW = nodes_[5]->output_buffer; - if (!pleW.is_cq() || (pleW.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) || !projW.is_cq()) return false; - ctx.ple = pleW.to_cq_matrix(); - ctx.proj = projW.to_cq_matrix(); - ctx.rms_weight = nodes_[4]->output_buffer.get_data(); - ctx.emb_scale = nodes_[7]->params.scalar; - ctx.ple_scale = nodes_[9]->params.scalar; - ctx.proj_scale = nodes_[14]->params.scalar; - ctx.final_scale = nodes_[22]->params.scalar; - ctx.rms_eps = nodes_[19]->params.epsilon; - if (!ctx.rms_weight || ctx.emb_scale == 0.0f || ctx.proj.N == 0) return false; - ctx.ok = true; - return true; -} - -bool CactusGraph::execute_gpu_fused() { - if (!cactus_metal_available()) return false; - const size_t n = nodes_.size(); - if (n < 3861) return false; - auto B = [&](size_t idx) -> BufferDesc& { return nodes_[idx]->output_buffer; }; - auto P = [&](size_t idx) -> void* { return B(idx).get_data(); }; - if (B(0).total_size != HID) return false; - if (nodes_[0]->op_type != OpType::INPUT || B(0).precision != Precision::FP16) return false; - g_gpu_argmax_valid = false; - - if (!G.init) { - - auto cq = [&](size_t idx){ return B(idx).to_cq_matrix(); }; - for (int l = 0; l < NL; ++l) { - LayerW& w = G.L[l]; - w.global = (l % 5 == 4); - w.hd = w.global ? 512 : 256; - w.shared = (l >= 15); - if (!w.shared) { - size_t b1 = 3 + 10*l, b2 = 334 + 6*l; - w.in_norm=P(b1); w.q_norm=P(b1+1); w.k_norm=P(b1+2); w.post_attn=P(b1+3); - w.pre_ffn=P(b1+4); w.gate=cq(b1+5); w.up=cq(b1+6); w.down=cq(b1+7); - w.post_ffn=P(b1+8); w.post_ple=P(b1+9); - w.q=cq(b2); w.k=cq(b2+1); w.v=cq(b2+2); w.o=cq(b2+3); w.ple_down=cq(b2+4); w.ple_up=cq(b2+5); - w.mlp = 6144; - } else { - int j = l - 15; size_t b1 = 153 + 9*j, b2 = 424 + 4*j; - w.in_norm=P(b1); w.q_norm=P(b1+1); w.post_attn=P(b1+2); w.pre_ffn=P(b1+3); - w.gate=cq(b1+4); w.up=cq(b1+5); w.down=cq(b1+6); w.post_ffn=P(b1+7); w.post_ple=P(b1+8); - w.q=cq(b2); w.o=cq(b2+1); w.ple_down=cq(b2+2); w.ple_up=cq(b2+3); - w.mlp = 12288; - } - w.scalar = P(505 + l); - } - G.vnorm = P(540); G.vnorm_g = P(541); - G.cosL = P(542); G.sinL = P(543); G.cosG = P(544); G.sinG = P(545); - G.final_norm = P(333); G.lm_head = B(504).to_cq_matrix(); - - std::vector cidx; - for (size_t i = 0; i < n && cidx.size() < 30; ++i) - if (nodes_[i]->op_type == OpType::KV_CACHE_STATE) cidx.push_back(i); - if (cidx.size() < 30) return false; - for (int l = 0; l < 15; ++l) { G.kc_idx[l] = cidx[2*l]; G.vc_idx[l] = cidx[2*l+1]; } - - auto sh = [&](size_t bytes){ void* p = cactus_metal_alloc_shared(bytes); if (p) std::memset(p,0,bytes); return p; }; - G.cos_s = sh(256*2); G.sin_s = sh(256*2); G.cos_g = sh(512*2); G.sin_g = sh(512*2); - G.amax = sh(3*sizeof(float)); - G.owner = this; - G.init = true; - } - if (G.owner != this) { - G.init = false; - return false; - } - - const bool fold = g_fe.ok; - - int pos; - if (fold) { - pos = g_fe.position; - } else { - G.hidden = P(0); G.ple = P(1); G.position = static_cast(P(2)); - if (!G.hidden || !G.ple || !G.position) return false; - pos = (int)G.position[0]; - } - for (int l = 0; l < 15; ++l) { G.L[l].kc = P(G.kc_idx[l]); G.L[l].vc = P(G.vc_idx[l]); } - for (int l = 15; l < NL; ++l) { int src = G.L[l].global ? 14 : 13; G.L[l].kc = G.L[src].kc; G.L[l].vc = G.L[src].vc; } - for (int l = 0; l < NL; ++l) if (!G.L[l].kc || !G.L[l].vc) return false; - - for (int l = 0; l < 15; ++l) { - const uint64_t* km = (const uint64_t*)G.L[l].kc; - if (G.L[l].global && km[0] + 1 > km[1]) return false; - } - - cactus_metal_session_begin(); - - auto encode_step = [&](int pos_, uint32_t tok, void** lg_out) -> int { - G.arena.reset(); - auto fresh = [&](size_t elems){ return G.arena.fresh(elems * 2); }; - - void* h = fresh(HID); - const void* pleBase; - if (fold) { - const int PK = (int)g_fe.proj.N; - void* pe = fresh(g_fe.ple.K); - bool e1 = cactus_metal_encode_embedding_ortho(h, tok, &G.lm_head, g_fe.emb_scale); - bool e2 = cactus_metal_encode_embedding_hadamard(pe, tok, &g_fe.ple); - if (!e1 || !e2) return 0; - void* pa = fresh(PK); cactus_metal_encode_scalar(2, pa, pe, PK, g_fe.ple_scale); - void* pj = fresh(PK); - if (!cactus_metal_encode_transform_gemv(pj, h, &g_fe.proj, nullptr) - && !cactus_metal_encode_quant_matmul(pj, h, &g_fe.proj)) return 0; - void* pjs= fresh(PK); cactus_metal_encode_scalar(2, pjs, pj, PK, g_fe.proj_scale); - void* plt= fresh(PK); - if (!cactus_metal_encode_rms_norm_add_scale(plt, pjs, g_fe.rms_weight, pa, NL, PLE_DIM, g_fe.rms_eps, g_fe.final_scale)) { - void* psum=fresh(PK); cactus_metal_encode_rms_norm_add(psum, pjs, g_fe.rms_weight, pa, NL, PLE_DIM, g_fe.rms_eps); - cactus_metal_encode_scalar(2, plt, psum, PK, g_fe.final_scale); - } - pleBase = plt; - } else { - std::memcpy(h, G.hidden, HID*2); - pleBase = G.ple; - } - - auto gather = [&](const void* tbl, void* dst, int hd){ std::memcpy(dst, (const char*)tbl + (size_t)pos_*hd*2, hd*2); }; - gather(G.cosL, G.cos_s, 256); gather(G.sinL, G.sin_s, 256); - gather(G.cosG, G.cos_g, 512); gather(G.sinG, G.sin_g, 512); - - void* xn = fresh(HID); - cactus_metal_encode_rms_norm(xn, h, G.L[0].in_norm, 1, HID, RMS_EPS); - - for (int l = 0; l < NL; ++l) { - - if (l > 0 && (l % FLUSH_LAYERS) == 0) cactus_metal_session_flush(); - LayerW& w = G.L[l]; - const int hd = w.hd, QD = NQH*hd; - const void* rc = w.global ? G.cos_g : G.cos_s; - const void* rs = w.global ? G.sin_g : G.sin_s; - uint64_t* km = (uint64_t*)w.kc; uint64_t* vm = (uint64_t*)w.vc; - size_t mx = km[1], ng = (hd+31)/32; - - void* q = fresh(QD); - void* attn = fresh(QD); - bool fattn = false; - const uint32_t Wn = w.global ? 0u : (uint32_t)(km[1] - km[4] - 1); - const uint32_t Sn = w.global ? 0u : (uint32_t)km[4]; - const uint32_t Rn = (Wn > Sn) ? (Wn - Sn) : 1u; - - if (!w.shared) { - void* k = fresh(hd); - void* v = fresh(hd); - const CactusQuantMatrix* Wqkv[3]={&w.q,&w.k,&w.v}; - void* yqkv[3]={q,k,v}; - void *cq=fresh(HID), *ck=fresh(HID), *cv=fresh(HID); - void* codes[3]={cq,ck,cv}; - if (cactus_metal_encode_transform_batch(xn, Wqkv, 3, codes)) { - const void* ccodes[3]={cq,ck,cv}; - if (!cactus_metal_encode_gemv_cat(yqkv, ccodes, Wqkv, 3)) { - cactus_metal_encode_gemv_precoded(q, cq, &w.q); - cactus_metal_encode_gemv_precoded(k, ck, &w.k); - cactus_metal_encode_gemv_precoded(v, cv, &w.v); - } - } else { - cactus_metal_encode_quant_matmul(q, xn, &w.q); - cactus_metal_encode_quant_matmul(k, xn, &w.k); - cactus_metal_encode_quant_matmul(v, xn, &w.v); - } - size_t clen = km[0]; - bool wrap = (!w.global && clen >= (size_t)Wn); - size_t slot = wrap ? (size_t)(Sn + ((clen - Sn) % Rn)) : clen; - size_t f_kv_end = wrap ? (size_t)Wn : std::min(clen + 1, (size_t)pos_ + 1); - fattn = cactus_metal_encode_attention_fused_i8(attn, q, k, v, - (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, - w.q_norm, w.k_norm, w.global ? G.vnorm_g : G.vnorm, rc, rs, - NQH, (uint32_t)hd, (uint32_t)hd, - 0u, (uint32_t)f_kv_end, (uint32_t)slot, 1u, RMS_EPS, 1.0f, - mx*hd, mx*hd, mx*ng*sizeof(float), mx*ng*sizeof(float)); - if (fattn) { - km[0] = clen + 1; vm[0] = clen + 1; - } else { - void* qr = fresh(QD); - void* kr = fresh(hd); - void* vn = fresh(hd); - bool okq = cactus_metal_encode_rms_norm_rope(qr, q, w.q_norm, rc, rs, NQH, hd, RMS_EPS); - bool okk = cactus_metal_encode_rms_norm_rope(kr, k, w.k_norm, rc, rs, 1, hd, RMS_EPS); - cactus_metal_encode_rms_norm(vn, v, w.global ? G.vnorm_g : G.vnorm, 1, hd, RMS_EPS); - if (!okq) { - void* qn = fresh(QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, NQH, hd, RMS_EPS); - cactus_metal_encode_rope(qr, qn, rc, rs, NQH, hd); - } - if (!okk) { - void* kn = fresh(hd); cactus_metal_encode_rms_norm(kn, k, w.k_norm, 1, hd, RMS_EPS); - cactus_metal_encode_rope(kr, kn, rc, rs, 1, hd); - } - cactus_metal_encode_kv_append_i8(kr, (char*)w.kc+64, (char*)w.kc+64+mx*hd, - 1, hd, (uint32_t)slot, 32, hd*2, mx*hd, mx*ng*sizeof(float)); - cactus_metal_encode_kv_append_i8(vn, (char*)w.vc+64, (char*)w.vc+64+mx*hd, - 1, hd, (uint32_t)slot, 32, hd*2, mx*hd, mx*ng*sizeof(float)); - km[0] = clen + 1; vm[0] = clen + 1; - size_t hist, total, kv_end; const void *knewp, *vnewp; - if (wrap) { - hist = Wn; total = Wn; knewp = nullptr; vnewp = nullptr; kv_end = Wn; - } else { - hist = clen; total = clen + 1; knewp = kr; vnewp = vn; - kv_end = std::min(total, (size_t)pos_ + 1); - } - cactus_metal_encode_attention_i8(attn, qr, knewp, vnewp, - (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, - NQH, 1, hd, hd, (uint32_t)hist, (uint32_t)total, 0u, (uint32_t)kv_end, 1.0f, - hist*hd, hist*hd, hist*ng*sizeof(float), hist*ng*sizeof(float)); - } - } else { - cactus_metal_encode_quant_matmul(q, xn, &w.q); - size_t clen = km[0]; - size_t s_kv_end = (!w.global && clen > (size_t)Wn) ? (size_t)Wn : std::min(clen, (size_t)pos_ + 1); - fattn = cactus_metal_encode_attention_fused_i8(attn, q, nullptr, nullptr, - (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, - w.q_norm, nullptr, nullptr, rc, rs, - NQH, (uint32_t)hd, (uint32_t)hd, - 0u, (uint32_t)s_kv_end, 0u, 0u, RMS_EPS, 1.0f, - mx*hd, mx*hd, mx*ng*sizeof(float), mx*ng*sizeof(float)); - if (!fattn) { - void* qr = fresh(QD); - if (!cactus_metal_encode_rms_norm_rope(qr, q, w.q_norm, rc, rs, NQH, hd, RMS_EPS)) { - void* qn = fresh(QD); cactus_metal_encode_rms_norm(qn, q, w.q_norm, NQH, hd, RMS_EPS); - cactus_metal_encode_rope(qr, qn, rc, rs, NQH, hd); - } - size_t hist, total, kv_end; const void *knewp, *vnewp; - if (!w.global && clen > (size_t)Wn) { - hist = Wn; total = Wn; knewp = nullptr; vnewp = nullptr; kv_end = Wn; - } else { - hist = clen; total = clen; knewp = qr; vnewp = qr; - kv_end = std::min(clen, (size_t)pos_ + 1); - } - cactus_metal_encode_attention_i8(attn, qr, knewp, vnewp, - (char*)w.kc+64, (char*)w.vc+64, (char*)w.kc+64+mx*hd, (char*)w.vc+64+mx*hd, - NQH, 1, hd, hd, (uint32_t)hist, (uint32_t)total, 0u, (uint32_t)kv_end, 1.0f, - hist*hd, hist*hd, hist*ng*sizeof(float), hist*ng*sizeof(float)); - } - } - void* o = fresh(HID); - cactus_metal_encode_quant_matmul(o, attn, &w.o); - void* h1 = fresh(HID); - void* xn2 = fresh(HID); - if (!cactus_metal_encode_rms_norm_add_rms(h1, xn2, o, w.post_attn, h, w.pre_ffn, 1, HID, RMS_EPS, 1.0f)) { - cactus_metal_encode_rms_norm_add(h1, o, w.post_attn, h, 1, HID, RMS_EPS); - cactus_metal_encode_rms_norm(xn2, h1, w.pre_ffn, 1, HID, RMS_EPS); - } - h = h1; - - const int M = w.mlp; - void* gate= fresh(M); - void* up = fresh(M); - const CactusQuantMatrix* Wgu[2]={&w.gate,&w.up}; - { - void* ygu[2]={gate,up}; - void *cg=fresh(HID), *cu=fresh(HID); void* cgu[2]={cg,cu}; - if (cactus_metal_encode_transform_batch(xn2, Wgu, 2, cgu)) { - const void* gcodes[2]={cg,cu}; - if (!cactus_metal_encode_gemv_cat(ygu, gcodes, Wgu, 2)) { - cactus_metal_encode_gemv_precoded(gate, cg, &w.gate); - cactus_metal_encode_gemv_precoded(up, cu, &w.up); - } - } else { - cactus_metal_encode_quant_matmul(gate, xn2, &w.gate); - cactus_metal_encode_quant_matmul(up, xn2, &w.up); - } - } - void* mo = fresh(HID); - { - void* gcode = fresh(M); - if (!cactus_metal_encode_swiglu_transform(gcode, gate, up, &w.down, GATE_SCALE) - || !cactus_metal_encode_gemv_precoded(mo, gcode, &w.down)) { - void* g3 = fresh(M); cactus_metal_encode_swiglu(g3, gate, up, M, GATE_SCALE); - cactus_metal_encode_quant_matmul(mo, g3, &w.down); - } - } - void* h2 = fresh(HID); cactus_metal_encode_rms_norm_add(h2, mo, w.post_ffn, h, 1, HID, RMS_EPS); h = h2; - - void* pm = fresh(PLE_DIM); - void* pu = fresh(HID); - { - const void* pls = (const char*)pleBase + (size_t)l*PLE_DIM*2; - if (!cactus_metal_encode_transform_gemv(pm, h, &w.ple_down, pls) - || !cactus_metal_encode_transform_gemv(pu, pm, &w.ple_up, nullptr)) { - void* ps = fresh(PLE_DIM); cactus_metal_encode_quant_matmul(ps, h, &w.ple_down); - cactus_metal_encode_swiglu(pm, ps, pls, PLE_DIM, 1.0f); - cactus_metal_encode_quant_matmul(pu, pm, &w.ple_up); - } - } - float ls = (float)(*(const __fp16*)w.scalar); - void* h3 = fresh(HID); - void* xnn = fresh(HID); - const void* next_w = (l+1 < NL) ? G.L[l+1].in_norm : G.final_norm; - if (cactus_metal_encode_rms_norm_add_rms(h3, xnn, pu, w.post_ple, h, next_w, 1, HID, RMS_EPS, ls)) { - h = h3; xn = xnn; - } else if (cactus_metal_encode_rms_norm_add_scale(h3, pu, w.post_ple, h, 1, HID, RMS_EPS, ls)) { - cactus_metal_encode_rms_norm(xnn, h3, next_w, 1, HID, RMS_EPS); - h = h3; xn = xnn; - } else { - cactus_metal_encode_rms_norm_add(h3, pu, w.post_ple, h, 1, HID, RMS_EPS); - void* h4 = fresh(HID); cactus_metal_encode_scalar(2, h4, h3, HID, ls); - cactus_metal_encode_rms_norm(xnn, h4, next_w, 1, HID, RMS_EPS); - h = h4; xn = xnn; - } - } - - size_t V = B(3860).total_size; - void* fn = xn; - void* code = fresh(HID); - void* lg = fresh(V); - *lg_out = lg; - if (cactus_metal_encode_quant_matmul_ortho(lg, fn, code, &G.lm_head)) { - if (!cactus_metal_encode_softcap(lg, lg, V, SOFTCAP)) { - cactus_metal_encode_scalar(3, lg, lg, V, SOFTCAP); - cactus_metal_encode_unary(1, lg, lg, V); - cactus_metal_encode_scalar(2, lg, lg, V, SOFTCAP); - } - bool adjusted = false; - if (sampling_needs_adjust()) { - adjusted = cactus_metal_encode_adjust_logits(lg, V, - g_samp.recent.data(), (uint32_t)g_samp.recent.size(), - g_samp.suppressed, g_samp.rep_penalty); - if (!adjusted) return 0; - } - g_step_adjusted = adjusted; - const float* bias = g_samp.active ? update_bias_dense(V) : nullptr; - if (!cactus_metal_encode_argmax(lg, (uint32_t)V, G.amax, bias)) return 0; - g_step_amax_biased = (bias != nullptr); - return 1; - } - cactus_metal_session_sync(); - cactus_quant_matmul(&G.lm_head, (const __fp16*)fn, 1u, (__fp16*)lg); - __fp16* L=(__fp16*)lg; for (size_t i=0;i +#include +#include +#include + +namespace { + +bool is_alias_op(OpType op) { + return op == OpType::VIEW || op == OpType::RESHAPE || op == OpType::FLATTEN; +} + +bool is_noop_transpose(const GraphNode& nd) { + if (nd.op_type != OpType::TRANSPOSE) return false; + size_t real = 0; + for (size_t d : nd.output_buffer.shape) if (d != 1) ++real; + return real <= 1; +} + +bool is_same_cast(const GraphNode& nd, const std::vector>& nodes, + const std::unordered_map& map) { + if (nd.op_type != OpType::PRECISION_CAST || nd.input_ids.empty()) return false; + auto it = map.find(nd.input_ids[0]); + return it != map.end() && nodes[it->second]->output_buffer.precision == nd.output_buffer.precision; +} + +} + +struct MetalCluster { + int rule = 0; + size_t a0 = 0, a1 = 0, a2 = 0, a3 = 0, a4 = 0, a5 = 0; + size_t b0 = 0, b1 = 0, b2 = 0, b3 = 0, b4 = 0; + float f0 = 0.0f, f1 = 0.0f; + uint32_t u0 = 0, u1 = 0; + void* s0 = nullptr; + void* s1 = nullptr; + void* sc[3] = {nullptr, nullptr, nullptr}; +}; + +bool cactus_graph_metal_tail(void* logits, size_t vocab); +struct FusedEmbedCtx; +const FusedEmbedCtx* cactus_graph_fused_embed(); +bool cactus_graph_metal_fold_prologue(void* h_buf, void* ple_buf, void* pos_buf, + const CactusQuantMatrix* lm_head, size_t nl, size_t ple_dim); + +struct MetalFusePlan { + bool built = false; + std::vector action; + std::vector clusters; + long fold_h = -1, fold_ple = -1, fold_pos = -1, fold_w = -1; + size_t fold_nl = 0, fold_pd = 0; + std::vector exec_list; + std::vector arena_off; + char* arena_base = nullptr; +}; + +MetalFusePlan* cactus_metal_plan_build( + const std::vector>& nodes, + const std::unordered_map& map) { + const size_t n = nodes.size(); + auto plan = new MetalFusePlan(); + plan->built = true; + plan->action.assign(n, -1); + + auto idxof = [&](size_t id) -> long { + auto it = map.find(id); + return it == map.end() ? -1 : (long)it->second; + }; + std::vector> cons(n); + for (size_t i = 0; i < n; ++i) + for (size_t id : nodes[i]->input_ids) { long j = idxof(id); if (j >= 0) cons[(size_t)j].push_back(i); } + + auto passthrough = [&](const GraphNode& nd) { + return is_alias_op(nd.op_type) || is_same_cast(nd, nodes, map) || is_noop_transpose(nd); + }; + auto up = [&](long i) -> long { + while (i >= 0) { + const GraphNode& nd = *nodes[(size_t)i]; + if (passthrough(nd) && !nd.input_ids.empty()) { + i = idxof(nd.input_ids[0]); + continue; + } + return i; + } + return -1; + }; + auto up_in = [&](size_t i, size_t k) -> long { + if (k >= nodes[i]->input_ids.size()) return -1; + return up(idxof(nodes[i]->input_ids[k])); + }; + auto deep_f16_source = [&](long i) -> long { + long cur = i; + while (cur >= 0) { + const GraphNode& nd = *nodes[(size_t)cur]; + if ((passthrough(nd) || nd.op_type == OpType::PRECISION_CAST) && !nd.input_ids.empty()) { + cur = idxof(nd.input_ids[0]); + continue; + } + break; + } + if (cur >= 0 && nodes[(size_t)cur]->output_buffer.precision == Precision::FP16) return cur; + return i; + }; + auto collect_chain = [&](size_t from, long to, std::vector& out_chain) { + long i = (long)from; + while (i >= 0 && i != to) { + const GraphNode& nd = *nodes[(size_t)i]; + if (!passthrough(nd)) break; + out_chain.push_back((size_t)i); + i = nd.input_ids.empty() ? -1 : idxof(nd.input_ids[0]); + } + }; + auto sole_use = [&](long i, const std::vector& allowed) -> bool { + if (i < 0) return false; + for (size_t c : cons[(size_t)i]) { + bool ok = false; + for (size_t a : allowed) if (a == c) { ok = true; break; } + if (!ok) { + const GraphNode& cn = *nodes[c]; + if (is_alias_op(cn.op_type) || is_same_cast(cn, nodes, map) || is_noop_transpose(cn)) { + std::vector sub = allowed; + sub.push_back(c); + bool subok = true; + for (size_t cc : cons[c]) { + bool found = false; + for (size_t a : allowed) if (a == cc) { found = true; break; } + if (!found) { subok = false; break; } + } + if (!subok) return false; + } else return false; + } + } + return true; + }; + + auto match_rope = [&](long add_i, long* x_out, long* cos_out, long* sin_out, + std::vector& cover) -> bool { + if (add_i < 0 || nodes[(size_t)add_i]->op_type != OpType::ADD) return false; + long m0 = up_in((size_t)add_i, 0), m1 = up_in((size_t)add_i, 1); + if (m0 < 0 || m1 < 0) return false; + if (nodes[(size_t)m0]->op_type != OpType::MULTIPLY) std::swap(m0, m1); + if (m0 < 0 || m1 < 0) return false; + if (nodes[(size_t)m0]->op_type != OpType::MULTIPLY || + nodes[(size_t)m1]->op_type != OpType::MULTIPLY) return false; + long rot = up_in((size_t)m1, 0); + long sinb = up_in((size_t)m1, 1); + if (rot < 0 || nodes[(size_t)rot]->op_type != OpType::CAT) { std::swap(rot, sinb); } + if (rot < 0 || sinb < 0 || nodes[(size_t)rot]->op_type != OpType::CAT) { + std::swap(m0, m1); + rot = up_in((size_t)m1, 0); sinb = up_in((size_t)m1, 1); + if (rot < 0 || nodes[(size_t)rot]->op_type != OpType::CAT) { std::swap(rot, sinb); } + if (rot < 0 || sinb < 0 || nodes[(size_t)rot]->op_type != OpType::CAT) return false; + } + long x = up_in((size_t)m0, 0), cosb = up_in((size_t)m0, 1); + if (x < 0 || cosb < 0) return false; + if (nodes[(size_t)rot]->input_ids.size() != 2) return false; + long neg = up_in((size_t)rot, 0), lo = up_in((size_t)rot, 1); + if (neg < 0 || lo < 0) return false; + if (nodes[(size_t)neg]->op_type != OpType::SCALAR_MULTIPLY || + nodes[(size_t)neg]->params.scalar != -1.0f) return false; + long hi = up_in((size_t)neg, 0); + if (hi < 0 || nodes[(size_t)hi]->op_type != OpType::SLICE || + nodes[(size_t)lo]->op_type != OpType::SLICE) return false; + long xs1 = up_in((size_t)hi, 0), xs2 = up_in((size_t)lo, 0); + if (xs1 != x || xs2 != x) { + long xa = up_in((size_t)m0, 1); + if (xs1 == xa && xs2 == xa) { x = xa; cosb = up_in((size_t)m0, 0); } + else return false; + } + *x_out = x; *cos_out = cosb; *sin_out = sinb; + cover.push_back((size_t)add_i); cover.push_back((size_t)m0); cover.push_back((size_t)m1); + cover.push_back((size_t)rot); cover.push_back((size_t)neg); + cover.push_back((size_t)hi); cover.push_back((size_t)lo); + return true; + }; + + auto add_cluster = [&](MetalCluster c, size_t anchor, const std::vector& cover) { + int32_t cid = (int32_t)plan->clusters.size(); + plan->clusters.push_back(c); + for (size_t v : cover) if (v != anchor) plan->action[v] = -2; + plan->action[anchor] = cid; + }; + + struct AttnCand { + MetalCluster c; + size_t anchor; + std::vector cover; + size_t kcache; + long kapp, vapp; + }; + std::vector cands; + + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd = *nodes[i]; + + if (nd.op_type == OpType::ATTENTION_CACHED && nd.input_ids.size() >= 5) { + const BufferDesc& qb = nodes[(size_t)idxof(nd.input_ids[0])]->output_buffer; + if (qb.shape.size() < 4 || qb.shape[0] != 1 || qb.shape[1] != 1) continue; + uint32_t nqh = (uint32_t)qb.shape[2], hd = (uint32_t)qb.shape[3]; + long qk = idxof(nd.input_ids[1]); + long vk = idxof(nd.input_ids[2]); + long kcache = idxof(nd.input_ids[3]); + long vcache = idxof(nd.input_ids[4]); + if (qk < 0 || vk < 0 || kcache < 0 || vcache < 0) continue; + + std::vector cover; + long q_rope = up_in(i, 0); + long qx = -1, qcos = -1, qsin = -1; + if (!match_rope(q_rope, &qx, &qcos, &qsin, cover)) continue; + long qnorm = qx; + if (qnorm < 0 || nodes[(size_t)qnorm]->op_type != OpType::RMS_NORM) continue; + long qraw = up_in((size_t)qnorm, 0); + long qw = up_in((size_t)qnorm, 1); + if (qraw < 0 || qw < 0) continue; + cover.push_back((size_t)qnorm); + + long k_rope = up(qk); + long kx = -1, kcos = -1, ksin = -1; + std::vector kcover; + long knorm = -1, kraw = -1, kw = -1, vnorm = -1, vraw = -1, vw = -1; + long kapp = -1, vapp = -1; + bool has_new = false; + if (match_rope(k_rope, &kx, &kcos, &ksin, kcover)) { + knorm = kx; + if (knorm >= 0 && nodes[(size_t)knorm]->op_type == OpType::RMS_NORM) { + kraw = up_in((size_t)knorm, 0); + kw = up_in((size_t)knorm, 1); + long vn = up(vk); + if (kraw >= 0 && kw >= 0 && vn >= 0 && nodes[(size_t)vn]->op_type == OpType::RMS_NORM) { + vnorm = vn; + vraw = up_in((size_t)vn, 0); + vw = up_in((size_t)vn, 1); + for (size_t c = 0; c < n; ++c) { + if (nodes[c]->op_type != OpType::KV_CACHE_APPEND || nodes[c]->input_ids.size() < 2) continue; + long src = up(idxof(nodes[c]->input_ids[0])); + long cache = idxof(nodes[c]->input_ids[1]); + if (src == k_rope && cache == kcache) kapp = (long)c; + if (src == (long)vnorm && cache == vcache) vapp = (long)c; + } + if (kapp >= 0 && vapp >= 0 && vraw >= 0 && vw >= 0) has_new = true; + } + } + } + bool shared = false; + if (has_new) { + for (auto& pc : cands) + if (pc.kapp == kapp || pc.kcache == (size_t)kcache) { shared = true; break; } + } + + AttnCand cand; + cand.anchor = i; + cand.kcache = (size_t)kcache; + cand.kapp = kapp; cand.vapp = vapp; + MetalCluster& c = cand.c; + c.rule = 1; + c.b0 = (size_t)qcos; c.b1 = (size_t)qsin; + c.b2 = (size_t)kcache; c.b3 = (size_t)vcache; + c.b4 = i; + c.a0 = (size_t)qraw; c.a3 = (size_t)deep_f16_source(qw); + c.f0 = nodes[(size_t)qnorm]->params.epsilon; + c.f1 = nd.params.scale != 0.0f ? nd.params.scale : 1.0f / std::sqrt((float)hd); + c.u0 = nqh; c.u1 = hd; + cand.cover = cover; + collect_chain((size_t)idxof(nd.input_ids[0]), q_rope, cand.cover); + if (has_new && !shared) { + c.a1 = (size_t)kraw; c.a2 = (size_t)vraw; + c.a4 = (size_t)deep_f16_source(kw); c.a5 = (size_t)deep_f16_source(vw); + c.u1 |= 0x10000u; + cand.cover.insert(cand.cover.end(), kcover.begin(), kcover.end()); + cand.cover.push_back((size_t)knorm); + cand.cover.push_back((size_t)vnorm); + cand.cover.push_back((size_t)kapp); + cand.cover.push_back((size_t)vapp); + collect_chain((size_t)idxof(nd.input_ids[1]), k_rope, cand.cover); + collect_chain((size_t)idxof(nd.input_ids[2]), vnorm, cand.cover); + collect_chain((size_t)idxof(nodes[(size_t)kapp]->input_ids[0]), k_rope, cand.cover); + collect_chain((size_t)idxof(nodes[(size_t)vapp]->input_ids[0]), vnorm, cand.cover); + } else if (!shared) { + continue; + } + cands.push_back(std::move(cand)); + continue; + } + + if (nd.op_type == OpType::MATMUL && nd.input_ids.size() >= 2) { + long r = idxof(nd.input_ids[1]); + const BufferDesc* rb = r >= 0 ? &nodes[(size_t)r]->output_buffer : nullptr; + size_t M = 1; + for (size_t d = 0; d + 1 < nd.output_buffer.shape.size(); ++d) M *= nd.output_buffer.shape[d]; + if (rb && M == 1 && PrecisionTraits::is_cq(rb->precision) && rb->group_size > 0 + && PrecisionTraits::cq_bits(rb->precision) == 4 + && !(rb->cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) { + long mul = up_in(i, 0); + if (mul >= 0 && nodes[(size_t)mul]->op_type == OpType::MULTIPLY && sole_use(mul, {i})) { + long g0 = up_in((size_t)mul, 0), u0v = up_in((size_t)mul, 1); + float gsc = 1.0f; + long smul = -1; + for (int side = 0; side < 2; ++side) { + long gel = side == 0 ? g0 : u0v; + long other = side == 0 ? u0v : g0; + long sm = -1; + float sc_val = 1.0f; + if (gel >= 0 && nodes[(size_t)gel]->op_type == OpType::SCALAR_MULTIPLY + && sole_use(gel, {(size_t)mul})) { + sm = gel; + sc_val = nodes[(size_t)gel]->params.scalar; + gel = up_in((size_t)gel, 0); + } + if (gel >= 0 && other >= 0 && nodes[(size_t)gel]->op_type == OpType::GELU + && sole_use(gel, {sm >= 0 ? (size_t)sm : (size_t)mul})) { + long gate = up_in((size_t)gel, 0); + if (gate < 0) break; + if (sc_val == 1.0f && nodes[(size_t)gate]->op_type == OpType::MATMUL + && nodes[(size_t)gate]->input_ids.size() >= 2 + && sole_use(gate, {(size_t)gel})) { + long gr = idxof(nodes[(size_t)gate]->input_ids[1]); + const BufferDesc* grb = gr >= 0 ? &nodes[(size_t)gr]->output_buffer : nullptr; + size_t gM = 1; + for (size_t d = 0; d + 1 < nodes[(size_t)gate]->output_buffer.shape.size(); ++d) + gM *= nodes[(size_t)gate]->output_buffer.shape[d]; + if (grb && gM == 1 && PrecisionTraits::is_cq(grb->precision) + && grb->group_size == 128 && PrecisionTraits::cq_bits(grb->precision) == 4 + && !(grb->cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) { + long gx = up_in((size_t)gate, 0); + if (gx >= 0) { + MetalCluster c7; + c7.rule = 7; + c7.a0 = (size_t)gx; + c7.a1 = (size_t)other; + c7.b0 = (size_t)gate; + c7.b4 = i; + c7.s0 = cactus_metal_alloc_shared(nodes[(size_t)gate]->output_buffer.total_size * 2); + std::vector cover; + cover.push_back((size_t)mul); + cover.push_back((size_t)gel); + cover.push_back((size_t)gate); + collect_chain((size_t)idxof(nd.input_ids[0]), mul, cover); + collect_chain((size_t)idxof(nodes[(size_t)gel]->input_ids[0]), gate, cover); + const GraphNode& rn = *nodes[(size_t)other]; + if (rn.op_type == OpType::INDEX && !rn.input_ids.empty() + && sole_use(other, {(size_t)mul})) { + long ib = idxof(rn.input_ids[0]); + if (ib >= 0) { + size_t rowlen = rn.output_buffer.total_size; + c7.a1 = (size_t)ib; + c7.u1 = (uint32_t)(rn.params.index_value * rowlen * 2); + c7.u0 = 1u; + cover.push_back((size_t)other); + } + } + add_cluster(c7, i, cover); + break; + } + } + } + MetalCluster c; + c.rule = 4; + c.a0 = (size_t)gate; + c.a1 = (size_t)other; + c.b4 = i; + c.f0 = sc_val; + const BufferDesc& gb = nodes[(size_t)gate]->output_buffer; + c.s0 = cactus_metal_alloc_shared(gb.total_size * 2); + std::vector cover; + cover.push_back((size_t)mul); + cover.push_back((size_t)gel); + if (sm >= 0) cover.push_back((size_t)sm); + collect_chain((size_t)idxof(nd.input_ids[0]), mul, cover); + add_cluster(c, i, cover); + gsc = sc_val; smul = sm; + break; + } + } + (void)gsc; (void)smul; + if (plan->action[i] >= 0) continue; + } + } + } + + if (nd.op_type == OpType::SCALAR_MULTIPLY && !nd.output_buffer.shape.empty() + && nd.output_buffer.shape.back() >= 32768 + && nd.output_buffer.total_size == nd.output_buffer.shape.back()) { + long th = up_in(i, 0); + if (th >= 0 && nodes[(size_t)th]->op_type == OpType::TANH) { + long dv = up_in((size_t)th, 0); + if (dv >= 0 && nodes[(size_t)dv]->op_type == OpType::SCALAR_DIVIDE + && nodes[(size_t)dv]->params.scalar == nd.params.scalar) { + long mm = up_in((size_t)dv, 0); + if (mm >= 0 && nodes[(size_t)mm]->op_type == OpType::MATMUL + && nodes[(size_t)mm]->input_ids.size() >= 2) { + long wnode = idxof(nodes[(size_t)mm]->input_ids[1]); + long xn = up_in((size_t)mm, 0); + if (wnode >= 0 && xn >= 0) { + const BufferDesc& wb = nodes[(size_t)wnode]->output_buffer; + if (PrecisionTraits::is_cq(wb.precision) && wb.group_size > 0 + && (wb.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) + && sole_use(mm, {(size_t)dv}) && sole_use(dv, {(size_t)th}) + && sole_use(th, {i})) { + MetalCluster c; + c.rule = 5; + c.a0 = (size_t)xn; c.a1 = (size_t)wnode; c.b4 = i; + c.f0 = nd.params.scalar; + c.s0 = cactus_metal_alloc_shared(wb.shape.empty() ? 0 : wb.to_cq_matrix().K * 2); + std::vector cover; + cover.push_back((size_t)mm); + cover.push_back((size_t)dv); + cover.push_back((size_t)th); + collect_chain((size_t)idxof(nodes[(size_t)dv]->input_ids[0]), mm, cover); + collect_chain((size_t)idxof(nd.input_ids[0]), th, cover); + add_cluster(c, i, cover); + continue; + } + } + } + } + } + } + + if (nd.op_type == OpType::ADD_CLIPPED && nd.input_ids.size() == 2) { + long n0 = up_in(i, 0), n1 = up_in(i, 1); + long norm = -1, res = -1; + if (n0 >= 0 && nodes[(size_t)n0]->op_type == OpType::RMS_NORM) { norm = n0; res = n1; } + else if (n1 >= 0 && nodes[(size_t)n1]->op_type == OpType::RMS_NORM) { norm = n1; res = n0; } + if (norm >= 0 && res >= 0 && sole_use(norm, {i})) { + long src = up_in((size_t)norm, 0); + long w = up_in((size_t)norm, 1); + auto f16 = [&](long j) { + return j >= 0 && nodes[(size_t)j]->output_buffer.precision == Precision::FP16; + }; + if (w >= 0) w = deep_f16_source(w); + bool one_row = src >= 0 && !nodes[(size_t)src]->output_buffer.shape.empty() + && nodes[(size_t)src]->output_buffer.total_size == + nodes[(size_t)src]->output_buffer.shape.back(); + if (src >= 0 && w >= 0 && one_row && f16((long)i) && f16(src) && f16(res) && f16(w) && f16(norm)) { + MetalCluster c; + c.rule = 3; + c.a0 = (size_t)src; c.a1 = (size_t)w; c.a2 = (size_t)res; + c.b4 = i; + c.f0 = nodes[(size_t)norm]->params.epsilon; + std::vector cover; + cover.push_back((size_t)norm); + size_t h_node = i; + if (cons[i].size() == 1) { + long mt = (long)cons[i][0]; + while (mt >= 0 && passthrough(*nodes[(size_t)mt]) && cons[(size_t)mt].size() == 1) + mt = (long)cons[(size_t)mt][0]; + if (mt >= 0 && nodes[(size_t)mt]->op_type == OpType::MULTIPLY + && nodes[(size_t)mt]->input_ids.size() == 2 + && nodes[(size_t)mt]->output_buffer.precision == Precision::FP16) { + long o0 = up_in((size_t)mt, 0), o1 = up_in((size_t)mt, 1); + long scn = (o0 == (long)i || (o0 >= 0 && up(o0) == (long)i)) ? o1 : o0; + long base = scn == o1 ? o0 : o1; + if (base >= 0 && scn >= 0 + && nodes[(size_t)scn]->output_buffer.total_size == 1 + && nodes[(size_t)scn]->output_buffer.precision == Precision::FP16) { + c.b1 = (size_t)scn; + c.u1 = 1u; + collect_chain(cons[i][0], (long)i, cover); + cover.push_back((size_t)mt); + h_node = (size_t)mt; + c.b4 = h_node; + cover.push_back(i); + } + } + } + long next_norm = -1; + for (size_t cc : cons[h_node]) { + long t = (long)cc; + while (t >= 0 && passthrough(*nodes[(size_t)t])) { + if (cons[(size_t)t].size() != 1) { t = -1; break; } + t = (long)cons[(size_t)t][0]; + } + if (t >= 0 && nodes[(size_t)t]->op_type == OpType::RMS_NORM + && up_in((size_t)t, 0) == (long)h_node + && nodes[(size_t)t]->output_buffer.precision == Precision::FP16) { + next_norm = t; + break; + } + } + if (next_norm >= 0) { + long nw = deep_f16_source(up_in((size_t)next_norm, 1)); + size_t ndim = nodes[(size_t)next_norm]->output_buffer.shape.empty() ? 0 + : nodes[(size_t)next_norm]->output_buffer.shape.back(); + size_t adim = nodes[i]->output_buffer.shape.empty() ? 0 + : nodes[i]->output_buffer.shape.back(); + if (nw >= 0 && ndim == adim && f16(nw)) { + c.u0 = 1u; + c.a4 = (size_t)nw; + c.a5 = (size_t)next_norm; + c.s1 = cactus_metal_alloc_shared(nodes[(size_t)next_norm]->output_buffer.total_size * 2); + cover.push_back((size_t)next_norm); + long ch = idxof(nodes[(size_t)next_norm]->input_ids[0]); + collect_chain((size_t)ch, (long)h_node, cover); + } + } + add_cluster(c, h_node, cover); + continue; + } + } + } + } + { + std::unordered_map> groups; + std::vector order; + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + const GraphNode& mn = *nodes[i]; + if (mn.op_type != OpType::MATMUL || mn.input_ids.size() < 2) continue; + long r = idxof(mn.input_ids[1]); + if (r < 0) continue; + const BufferDesc& rb = nodes[(size_t)r]->output_buffer; + size_t M = 1; + for (size_t d = 0; d + 1 < mn.output_buffer.shape.size(); ++d) M *= mn.output_buffer.shape[d]; + if (M != 1 || !PrecisionTraits::is_cq(rb.precision) || rb.group_size != 128 + || PrecisionTraits::cq_bits(rb.precision) != 4 + || (rb.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) continue; + long src = up_in(i, 0); + if (src < 0) continue; + auto it = groups.find(src); + if (it == groups.end()) { groups[src] = {i}; order.push_back(src); } + else it->second.push_back(i); + } + for (long src : order) { + auto& mms = groups[src]; + if (mms.size() < 2) continue; + for (size_t base = 0; base + 1 < mms.size(); base += 3) { + size_t cnt = std::min(3, mms.size() - base); + if (cnt < 2) break; + size_t anchor_i = mms[base + cnt - 1]; + bool ordered = true; + for (size_t mi = 0; mi < cnt && ordered; ++mi) + for (size_t cc : cons[mms[base + mi]]) + if (cc < anchor_i && plan->action[cc] != -2) { ordered = false; break; } + if (!ordered) continue; + MetalCluster c; + c.rule = 2; + c.a0 = (size_t)src; + c.b0 = mms[base]; + c.b1 = mms[base + 1]; + c.b2 = cnt > 2 ? mms[base + 2] : 0; + c.u0 = (uint32_t)cnt; + size_t maxK = 0; + for (size_t mi = 0; mi < cnt; ++mi) { + const BufferDesc& rb = nodes[(size_t)idxof(nodes[mms[base + mi]]->input_ids[1])]->output_buffer; + CactusQuantMatrix W = rb.to_cq_matrix(); + if (W.K > maxK) maxK = W.K; + } + for (uint32_t bi = 0; bi < c.u0; ++bi) + if (!c.sc[bi]) c.sc[bi] = cactus_metal_alloc_shared(maxK * 2); + std::vector cover; + add_cluster(c, mms[base + cnt - 1], cover); + for (size_t mi = 0; mi + 1 < cnt; ++mi) plan->action[mms[base + mi]] = -3; + } + } + } + + if (!cands.empty()) { + std::vector mark(n, 0); + for (auto& cd : cands) { + mark[cd.anchor] = 2; + for (size_t v : cd.cover) if (mark[v] == 0) mark[v] = 1; + } + for (size_t ri = n; ri-- > 0;) { + if (mark[ri]) continue; + const GraphNode& ndp = *nodes[ri]; + if (!(is_alias_op(ndp.op_type) || is_same_cast(ndp, nodes, map) || is_noop_transpose(ndp))) continue; + if (cons[ri].empty()) { mark[ri] = 3; continue; } + bool all_in = true; + for (size_t cc : cons[ri]) if (!mark[cc]) { all_in = false; break; } + if (all_in) mark[ri] = 3; + } + bool valid = true; + for (size_t v = 0; v < n && valid; ++v) { + if (mark[v] != 1 && mark[v] != 3) continue; + for (size_t cc : cons[v]) { + if (!mark[cc]) { + valid = false; + break; + } + } + } + if (valid) { + for (size_t v = 0; v < n; ++v) + if ((mark[v] == 1 || mark[v] == 3) && plan->action[v] == -1) plan->action[v] = -2; + for (auto& cd : cands) { + int32_t cid = (int32_t)plan->clusters.size(); + plan->clusters.push_back(cd.c); + plan->action[cd.anchor] = cid; + } + } + } + + for (auto& cl : plan->clusters) { + if (cl.rule != 5) continue; + const auto& sh0 = nodes[0]->output_buffer; + const auto& sh1 = n > 1 ? nodes[1]->output_buffer : nodes[0]->output_buffer; + const auto& sh2 = n > 2 ? nodes[2]->output_buffer : nodes[0]->output_buffer; + if (n > 2 && nodes[0]->op_type == OpType::INPUT && nodes[1]->op_type == OpType::INPUT + && nodes[2]->op_type == OpType::INPUT + && sh0.precision == Precision::FP16 && sh0.total_size > 0 + && sh1.precision == Precision::FP16 && sh1.shape.size() >= 2 + && sh2.total_size == 1) { + plan->fold_h = 0; + plan->fold_ple = 1; + plan->fold_pos = 2; + plan->fold_w = (long)cl.a1; + plan->fold_nl = sh1.shape[sh1.shape.size() - 2]; + plan->fold_pd = sh1.shape.back(); + } + break; + } + + plan->exec_list.reserve(n); + for (size_t i = 0; i < n; ++i) + if (plan->action[i] != -2 && nodes[i]->op_type != OpType::INPUT) + plan->exec_list.push_back((uint32_t)i); + + if (!plan->clusters.empty()) { + std::vector last_read(n, 0); + auto base_of = [&](size_t i) -> size_t { + size_t cur = i; + while (passthrough(*nodes[cur]) || plan->action[cur] == -2 || is_noop_transpose(*nodes[cur]) + || nodes[cur]->op_type == OpType::SLICE || nodes[cur]->op_type == OpType::INDEX) { + if (nodes[cur]->input_ids.empty()) break; + long j = idxof(nodes[cur]->input_ids[0]); + if (j < 0) break; + if (plan->action[cur] != -2 && !passthrough(*nodes[cur]) + && !is_noop_transpose(*nodes[cur])) { + bool aliasing = false; + const GraphNode& sn = *nodes[cur]; + if (sn.op_type == OpType::INDEX && sn.params.axis == 0) aliasing = true; + if (sn.op_type == OpType::SLICE) { + size_t ax = (size_t)sn.params.axis; + long jj = idxof(sn.input_ids[0]); + if (jj >= 0 && ax < nodes[(size_t)jj]->output_buffer.shape.size()) { + size_t outer = 1; + for (size_t d = 0; d < ax; ++d) outer *= nodes[(size_t)jj]->output_buffer.shape[d]; + if (outer == 1) aliasing = true; + } + } + if (!aliasing) break; + } + cur = (size_t)j; + } + return cur; + }; + for (size_t i = 0; i < n; ++i) { + for (size_t id : nodes[i]->input_ids) { + long j = idxof(id); + if (j >= 0) { + size_t b = base_of((size_t)j); + if (i > last_read[b]) last_read[b] = i; + } + } + } + for (auto& cl : plan->clusters) { + size_t anchor = cl.b4 ? cl.b4 : (cl.rule == 2 ? std::max(cl.b0, std::max(cl.b1, cl.b2)) : 0); + if (!anchor) continue; + for (size_t arg : {cl.a0, cl.a1, cl.a2, cl.a3, cl.a4, cl.a5, cl.b0, cl.b1, cl.b2, cl.b3}) { + if (arg == 0 || arg >= n) continue; + size_t b = base_of(arg); + if (anchor > last_read[b]) last_read[b] = anchor; + } + } + plan->arena_off.assign(n, -1); + struct Range { size_t off, size, free_at; }; + std::vector live; + size_t cursor = 0, peak = 0; + for (uint32_t ui : plan->exec_list) { + size_t i = ui; + const GraphNode& nd = *nodes[i]; + if (plan->action[i] == -2) continue; + if (nd.op_type == OpType::KV_CACHE_STATE || nd.op_type == OpType::CONV_CACHE_STATE + || nd.op_type == OpType::RECURRENT_CACHE_STATE || nd.op_type == OpType::KV_CACHE_APPEND + || nd.op_type == OpType::PERSISTENT) continue; + if (passthrough(nd) || is_noop_transpose(nd)) continue; + if (nd.op_type == OpType::SLICE || nd.op_type == OpType::INDEX) continue; + if (last_read[i] == 0) continue; + size_t need = (nd.output_buffer.byte_size + 255) & ~size_t(255); + if (need == 0 || need > (8u << 20)) continue; + bool placed = false; + for (auto& r : live) { + if (r.free_at < i && r.size >= need) { + plan->arena_off[i] = (long)r.off; + r.free_at = last_read[i]; + placed = true; + break; + } + } + if (!placed) { + plan->arena_off[i] = (long)cursor; + live.push_back({cursor, need, last_read[i]}); + cursor += need; + if (cursor > peak) peak = cursor; + } + } + if (peak > 0 && peak < (256u << 20)) + plan->arena_base = (char*)cactus_metal_alloc_shared(peak); + if (!plan->arena_base) plan->arena_off.assign(n, -1); + } + + return plan; +} + +void cactus_metal_plan_free(MetalFusePlan* p) { delete p; } + +void cactus_metal_plan_disable(MetalFusePlan* p) { + if (!p) return; + std::fill(p->action.begin(), p->action.end(), -1); + p->clusters.clear(); + p->fold_h = -1; + p->exec_list.clear(); + for (size_t i = 0; i < p->action.size(); ++i) p->exec_list.push_back((uint32_t)i); +} + +int32_t cactus_metal_plan_action(const MetalFusePlan* p, size_t i) { + return p && i < p->action.size() ? p->action[i] : -1; +} + +const std::vector* cactus_metal_plan_exec_list(const MetalFusePlan* p) { + return p ? &p->exec_list : nullptr; +} + +void* cactus_metal_plan_arena_ptr(const MetalFusePlan* p, size_t i) { + if (!p || !p->arena_base || i >= p->arena_off.size() || p->arena_off[i] < 0) return nullptr; + return p->arena_base + p->arena_off[i]; +} + +bool cactus_metal_plan_fold(MetalFusePlan* p, + const std::vector>& nodes) { + if (!p || p->fold_h < 0 || !cactus_graph_fused_embed()) return false; + void* h = nodes[(size_t)p->fold_h]->output_buffer.get_data(); + void* ple = nodes[(size_t)p->fold_ple]->output_buffer.get_data(); + void* pos = nodes[(size_t)p->fold_pos]->output_buffer.get_data(); + if (!h || !ple) return false; + CactusQuantMatrix W = nodes[(size_t)p->fold_w]->output_buffer.to_cq_matrix(); + return cactus_graph_metal_fold_prologue(h, ple, pos, &W, p->fold_nl, p->fold_pd); +} + +bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, + const std::vector>& nodes, + const std::unordered_map& map) { + MetalCluster& c = p->clusters[(size_t)cid]; + auto data = [&](size_t i) { return nodes[i]->output_buffer.get_data(); }; + switch (c.rule) { + case 1: { + GraphNode& anchor = *nodes[c.b4]; + BufferDesc& kc = nodes[c.b2]->output_buffer; + BufferDesc& vc = nodes[c.b3]->output_buffer; + uint64_t* km = reinterpret_cast(kc.get_data()); + uint64_t* vm = reinterpret_cast(vc.get_data()); + if (!km || !vm) return false; + size_t clen = km[0], mx = km[1], sink = km[4]; + uint32_t hd = c.u1 & 0xFFFFu, nqh = c.u0; + bool has_new = (c.u1 & 0x10000u) != 0; + size_t ng = (hd + 31) / 32; + size_t win = anchor.params.window_size; + bool sliding = win > 0; + uint32_t Wn = sliding ? (uint32_t)(mx - sink - 1) : 0u; + uint32_t Sn = sliding ? (uint32_t)sink : 0u; + uint32_t Rn = (Wn > Sn) ? (Wn - Sn) : 1u; + char* kb = (char*)kc.get_data(); + char* vb = (char*)vc.get_data(); + size_t slot = 0, kv_end; + if (has_new) { + bool wrap = sliding && clen >= (size_t)Wn; + slot = wrap ? (size_t)(Sn + ((clen - Sn) % Rn)) : clen; + kv_end = wrap ? (size_t)Wn : clen + 1; + } else { + kv_end = (sliding && clen > (size_t)Wn) ? (size_t)Wn : clen; + } + bool ok = cactus_metal_encode_attention_fused_i8( + anchor.output_buffer.get_data(), data(c.a0), + has_new ? data(c.a1) : nullptr, has_new ? data(c.a2) : nullptr, + kb + 64, vb + 64, kb + 64 + mx * hd, vb + 64 + mx * hd, + data(c.a3), has_new ? data(c.a4) : nullptr, has_new ? data(c.a5) : nullptr, + data(c.b0), data(c.b1), + nqh, hd, hd, + 0u, (uint32_t)kv_end, (uint32_t)slot, has_new ? 1u : 0u, + c.f0, c.f1, + mx * hd, mx * hd, mx * ng * sizeof(float), mx * ng * sizeof(float)); + if (!ok) return false; + if (has_new) { km[0] = clen + 1; vm[0] = clen + 1; } + return true; + } + case 2: { + size_t mm[3] = { c.b0, c.b1, c.b2 }; + CactusQuantMatrix W[3]; + void* outs[3]; + const CactusQuantMatrix* Wp[3]; + for (uint32_t bi = 0; bi < c.u0; ++bi) { + GraphNode& mn = *nodes[mm[bi]]; + auto it = map.find(mn.input_ids[1]); + if (it == map.end()) return false; + W[bi] = nodes[it->second]->output_buffer.to_cq_matrix(); + Wp[bi] = &W[bi]; + outs[bi] = mn.output_buffer.get_data(); + if (!outs[bi] || !c.sc[bi]) return false; + } + const void* x = nodes[c.a0]->output_buffer.get_data(); + void* codes[3] = { c.sc[0], c.sc[1], c.sc[2] }; + if (cactus_metal_encode_transform_batch(x, Wp, (int)c.u0, codes)) { + const void* ccodes[3] = { c.sc[0], c.sc[1], c.sc[2] }; + if (!cactus_metal_encode_gemv_cat(outs, ccodes, Wp, (int)c.u0)) { + for (uint32_t bi = 0; bi < c.u0; ++bi) + if (!cactus_metal_encode_gemv_precoded(outs[bi], c.sc[bi], Wp[bi])) return false; + } + return true; + } + for (uint32_t bi = 0; bi < c.u0; ++bi) + if (!cactus_metal_encode_quant_matmul(outs[bi], x, Wp[bi])) return false; + return true; + } + case 7: { + GraphNode& anchor = *nodes[c.b4]; + GraphNode& down = *nodes[c.b0]; + auto itd = map.find(down.input_ids[1]); + auto itu = map.find(anchor.input_ids[1]); + if (itd == map.end() || itu == map.end() || !c.s0) return false; + CactusQuantMatrix Wd = nodes[itd->second]->output_buffer.to_cq_matrix(); + CactusQuantMatrix Wu = nodes[itu->second]->output_buffer.to_cq_matrix(); + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* row = nodes[c.a1]->output_buffer.get_data(); + if (row && c.u0 == 1u) row = (const char*)row + c.u1; + void* out = anchor.output_buffer.get_data(); + if (!x || !row || !out) return false; + if (!cactus_metal_encode_transform_gemv(c.s0, x, &Wd, row)) return false; + if (!cactus_metal_encode_transform_gemv(out, c.s0, &Wu, nullptr) + && !cactus_metal_encode_quant_matmul(out, c.s0, &Wu)) return false; + return true; + } + case 4: { + GraphNode& anchor = *nodes[c.b4]; + auto it = map.find(anchor.input_ids[1]); + if (it == map.end() || !c.s0) return false; + CactusQuantMatrix W = nodes[it->second]->output_buffer.to_cq_matrix(); + void* out = anchor.output_buffer.get_data(); + const void* gate = nodes[c.a0]->output_buffer.get_data(); + const void* up = nodes[c.a1]->output_buffer.get_data(); + if (!out || !gate || !up) return false; + if (cactus_metal_encode_swiglu_transform(c.s0, gate, up, &W, c.f0) + && cactus_metal_encode_gemv_precoded(out, c.s0, &W)) return true; + size_t M = W.K; + if (!cactus_metal_encode_swiglu(c.s0, gate, up, M, c.f0)) return false; + return cactus_metal_encode_quant_matmul(out, c.s0, &W); + } + case 3: { + GraphNode& anchor = *nodes[c.b4]; + const BufferDesc& src = nodes[c.a0]->output_buffer; + size_t dim = anchor.output_buffer.shape.empty() ? 0 : anchor.output_buffer.shape.back(); + if (dim == 0) return false; + size_t rows = src.total_size / dim; + float out_scale = 1.0f; + if (c.u1 == 1u) { + const void* sp = nodes[c.b1]->output_buffer.get_data(); + if (!sp) return false; + out_scale = (float)*(const __fp16*)sp; + } + if (c.u0 == 1u && rows == 1) { + GraphNode& nn = *nodes[c.a5]; + if (!nn.output_buffer.get_data()) { + if (!c.s1) return false; + nn.output_buffer.set_external(c.s1); + } + if (cactus_metal_encode_rms_norm_add_rms( + anchor.output_buffer.get_data(), nn.output_buffer.get_data(), + src.get_data(), data(c.a1), data(c.a2), data(c.a4), + rows, dim, c.f0, out_scale)) return true; + } + if (c.u1 == 1u && c.u0 != 1u && rows == 1) { + if (cactus_metal_encode_rms_norm_add_scale( + anchor.output_buffer.get_data(), src.get_data(), data(c.a1), data(c.a2), + rows, dim, c.f0, out_scale)) return true; + return false; + } + if (c.u1 == 1u) return false; + if (c.u0 == 1u) { + GraphNode& nn = *nodes[c.a5]; + if (!nn.output_buffer.get_data()) { + if (!c.s1) return false; + nn.output_buffer.set_external(c.s1); + } + if (!cactus_metal_encode_rms_norm_add( + anchor.output_buffer.get_data(), src.get_data(), data(c.a1), data(c.a2), + rows, dim, c.f0)) return false; + return cactus_metal_encode_rms_norm( + nn.output_buffer.get_data(), anchor.output_buffer.get_data(), data(c.a4), + rows, dim, c.f0); + } + return cactus_metal_encode_rms_norm_add( + anchor.output_buffer.get_data(), src.get_data(), data(c.a1), data(c.a2), + rows, dim, c.f0); + } + case 5: { + GraphNode& anchor = *nodes[c.b4]; + const BufferDesc& wb = nodes[c.a1]->output_buffer; + CactusQuantMatrix W = wb.to_cq_matrix(); + void* lg = anchor.output_buffer.get_data(); + if (!c.s0 || !cactus_metal_encode_quant_matmul_ortho(lg, data(c.a0), c.s0, &W)) return false; + size_t V = anchor.output_buffer.total_size; + if (!cactus_metal_encode_softcap(lg, lg, V, c.f0)) { + cactus_metal_encode_scalar(3, lg, lg, V, c.f0); + cactus_metal_encode_unary(1, lg, lg, V); + cactus_metal_encode_scalar(2, lg, lg, V, c.f0); + } + return cactus_graph_metal_tail(lg, V); + } + default: return false; + } +} diff --git a/cactus-graph/src/metal_runtime.cpp b/cactus-graph/src/metal_runtime.cpp new file mode 100644 index 000000000..ba02d0174 --- /dev/null +++ b/cactus-graph/src/metal_runtime.cpp @@ -0,0 +1,198 @@ +#include "../cactus_graph.h" +#include "cactus_kernels.h" +#include "metal_backend.h" + +struct MetalFusePlan; +void cactus_metal_plan_free(MetalFusePlan* p); +#include +#include +#include +#include +#include + +namespace { + +static bool g_metal_argmax_valid = false; +static const float* g_metal_argmax_buf = nullptr; + +} + +static bool g_last_step_adjusted = false; +static bool g_last_amax_biased = false; +static bool g_prefill_consistent = false; + +void cactus_graph_mark_unadjusted() { + g_last_step_adjusted = false; + g_last_amax_biased = false; + g_metal_argmax_valid = false; +} + +void cactus_graph_set_prefill_consistent(bool on) { g_prefill_consistent = on; } + +bool cactus_graph_prefill_consistent() { return g_prefill_consistent; } + +struct GSampling { + bool active = false; + float rep_penalty = 1.0f; + std::vector recent; + const float* bias_dense = nullptr; + size_t bias_len = 0; + long long suppressed = -1; +}; +static GSampling g_samp; +static bool g_step_adjusted = false; +static bool g_step_amax_biased = false; +static float* g_bias_dense = nullptr; +static size_t g_bias_dense_n = 0; + +void cactus_graph_set_sampling(const uint32_t* recent, int n_recent, float rep_penalty, + const float* bias_dense, size_t bias_len, + long long suppressed) { + g_samp.active = true; + g_samp.rep_penalty = rep_penalty; + g_samp.recent.assign(recent, recent + (n_recent > 0 ? n_recent : 0)); + g_samp.bias_dense = bias_dense; + g_samp.bias_len = bias_dense ? bias_len : 0; + g_samp.suppressed = suppressed; +} +void cactus_graph_clear_sampling() { g_samp = GSampling(); } +bool cactus_graph_metal_adjusted() { return g_last_step_adjusted; } +bool cactus_graph_metal_argmax_biased() { return g_last_amax_biased; } + +static void* g_plan_amax = nullptr; +static bool g_plan_tail_pending = false; +static bool g_plan_tail_adjusted = false; +static bool g_plan_tail_biased = false; + +bool cactus_graph_metal_tail(void* logits, size_t vocab); +void cactus_graph_metal_tail_commit(); + +static const float* update_bias_dense(size_t vocab) { + if (!g_samp.bias_dense || g_samp.bias_len == 0) return nullptr; + if (!g_bias_dense || g_bias_dense_n < vocab) { + g_bias_dense = (float*)cactus_metal_alloc_shared(vocab * sizeof(float)); + if (!g_bias_dense) return nullptr; + g_bias_dense_n = vocab; + } + size_t n = g_samp.bias_len < vocab ? g_samp.bias_len : vocab; + std::memcpy(g_bias_dense, g_samp.bias_dense, n * sizeof(float)); + if (n < vocab) std::memset(g_bias_dense + n, 0, (vocab - n) * sizeof(float)); + return g_bias_dense; +} + +bool cactus_graph_metal_tail(void* logits, size_t vocab) { + g_plan_tail_pending = false; + bool adjusted = false; + if (g_samp.active && + ((g_samp.rep_penalty != 1.0f && !g_samp.recent.empty()) || g_samp.suppressed >= 0)) { + adjusted = cactus_metal_encode_adjust_logits(logits, vocab, + g_samp.recent.data(), (uint32_t)g_samp.recent.size(), + g_samp.suppressed, g_samp.rep_penalty); + if (!adjusted) return false; + } + if (!g_plan_amax) { + g_plan_amax = cactus_metal_alloc_shared(3 * sizeof(float)); + if (!g_plan_amax) return false; + } + const float* bias = g_samp.active ? update_bias_dense(vocab) : nullptr; + if (!cactus_metal_encode_argmax(logits, (uint32_t)vocab, g_plan_amax, bias)) return false; + g_plan_tail_pending = true; + g_plan_tail_adjusted = adjusted; + g_plan_tail_biased = (bias != nullptr); + return true; +} + +void cactus_graph_metal_tail_commit() { + if (!g_plan_tail_pending) return; + g_plan_tail_pending = false; + g_metal_argmax_buf = (const float*)g_plan_amax; + g_metal_argmax_valid = true; + g_last_step_adjusted = g_plan_tail_adjusted; + g_last_amax_biased = g_plan_tail_biased; +} + +static inline bool sampling_needs_adjust() { + return g_samp.active && + ((g_samp.rep_penalty != 1.0f && !g_samp.recent.empty()) || g_samp.suppressed >= 0); +} + +bool cactus_graph_metal_argmax(uint32_t* idx, float* best, float* second) { + if (!g_metal_argmax_valid || !g_metal_argmax_buf) return false; + g_metal_argmax_valid = false; + *best = g_metal_argmax_buf[0]; *second = g_metal_argmax_buf[1]; + *idx = (uint32_t)g_metal_argmax_buf[2]; + return true; +} + +static FusedEmbedCtx g_fe; +void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx) { + if (ctx && ctx->ok) g_fe = *ctx; else g_fe.ok = false; +} +const FusedEmbedCtx* cactus_graph_fused_embed() { return g_fe.ok ? &g_fe : nullptr; } + +bool cactus_graph_metal_fold_prologue(void* h_buf, void* ple_buf, void* pos_buf, + const CactusQuantMatrix* lm_head, size_t nl, size_t ple_dim) { + if (!g_fe.ok || !lm_head) return false; + static void* pe = nullptr; + static void* pa = nullptr; + static void* pj = nullptr; + static void* pjs = nullptr; + static size_t cap = 0; + const size_t PK = g_fe.proj.N; + if (cap < PK) { + pe = cactus_metal_alloc_shared(g_fe.ple.K * 2); + pa = cactus_metal_alloc_shared(PK * 2); + pj = cactus_metal_alloc_shared(PK * 2); + pjs = cactus_metal_alloc_shared(PK * 2); + cap = PK; + } + if (!pe || !pa || !pj || !pjs) return false; + uint32_t tok = (uint32_t)g_fe.token_id; + if (!cactus_metal_encode_embedding_ortho(h_buf, tok, lm_head, g_fe.emb_scale)) return false; + if (!cactus_metal_encode_embedding_hadamard(pe, tok, &g_fe.ple)) return false; + cactus_metal_encode_scalar(2, pa, pe, PK, g_fe.ple_scale); + if (!cactus_metal_encode_transform_gemv(pj, h_buf, &g_fe.proj, nullptr) + && !cactus_metal_encode_quant_matmul(pj, h_buf, &g_fe.proj)) return false; + cactus_metal_encode_scalar(2, pjs, pj, PK, g_fe.proj_scale); + if (!cactus_metal_encode_rms_norm_add_scale(ple_buf, pjs, g_fe.rms_weight, pa, nl, ple_dim, g_fe.rms_eps, g_fe.final_scale)) { + if (!cactus_metal_encode_rms_norm_add(pa, pjs, g_fe.rms_weight, pa, nl, ple_dim, g_fe.rms_eps)) return false; + cactus_metal_encode_scalar(2, ple_buf, pa, PK, g_fe.final_scale); + } + if (pos_buf) *(float*)pos_buf = (float)g_fe.position; + return true; +} + +bool CactusGraph::extract_ple_pathway(FusedEmbedCtx& ctx) const { + + if (nodes_.size() != 23) return false; + auto op = [&](size_t i){ return nodes_[i]->op_type; }; + if (op(3) != OpType::INPUT || op(4) != OpType::INPUT || op(5) != OpType::INPUT) return false; + if (op(8) != OpType::EMBEDDING || op(12) != OpType::MATMUL || op(19) != OpType::RMS_NORM) return false; + if (op(7) != OpType::SCALAR_MULTIPLY || op(9) != OpType::SCALAR_MULTIPLY || + op(14) != OpType::SCALAR_MULTIPLY || op(22) != OpType::SCALAR_MULTIPLY) return false; + const BufferDesc& pleW = nodes_[3]->output_buffer; + const BufferDesc& projW = nodes_[5]->output_buffer; + if (!pleW.is_cq() || (pleW.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) || !projW.is_cq()) return false; + ctx.ple = pleW.to_cq_matrix(); + ctx.proj = projW.to_cq_matrix(); + ctx.rms_weight = nodes_[4]->output_buffer.get_data(); + ctx.emb_scale = nodes_[7]->params.scalar; + ctx.ple_scale = nodes_[9]->params.scalar; + ctx.proj_scale = nodes_[14]->params.scalar; + ctx.final_scale = nodes_[22]->params.scalar; + ctx.rms_eps = nodes_[19]->params.epsilon; + if (!ctx.rms_weight || ctx.emb_scale == 0.0f || ctx.proj.N == 0) return false; + ctx.ok = true; + return true; +} + +void cactus_graph_on_destroy(const void* graph) { + (void)graph; + g_samp = GSampling(); + cactus_metal_invalidate_host_wraps(); +} + +CactusGraph::~CactusGraph() { + cactus_graph_on_destroy(this); + for (auto& kv : metal_plans_) cactus_metal_plan_free(kv.second); +} diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 78a3c9cba..d6383c0bf 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -31,6 +31,7 @@ void cactus_metal_invalidate_host_wraps(); void cactus_metal_session_end(); void* cactus_metal_alloc_shared(size_t bytes); +void* cactus_metal_alloc_pooled(size_t bytes); void cactus_metal_free_shared(void* contents); bool cactus_metal_encode_copy(void* out, const void* in, size_t bytes); diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 65fd53a38..fc2945226 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -260,11 +260,24 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, return { it->second, (size_t)(a - it->first) }; } size_t wraplen = (need + 16383u) & ~(size_t)16383u; - id b = [ctx().dev newBufferWithBytesNoCopy:(void*)base length:wraplen + uintptr_t lo = base, hi = base + wraplen; + auto ov = g_wrapped.lower_bound(lo); + if (ov != g_wrapped.begin()) { + auto prev = std::prev(ov); + if (prev->first + (size_t)prev->second.length > lo) ov = prev; + } + while (ov != g_wrapped.end() && ov->first < hi) { + if (ov->first + (size_t)ov->second.length > lo) { + if (ov->first < lo) lo = ov->first; + if (ov->first + (size_t)ov->second.length > hi) hi = ov->first + (size_t)ov->second.length; + ov = g_wrapped.erase(ov); + } else ++ov; + } + id b = [ctx().dev newBufferWithBytesNoCopy:(void*)lo length:(size_t)(hi - lo) options:MTLResourceStorageModeShared deallocator:nil]; if (!b) return { nil, 0 }; - g_wrapped[base] = b; - return { b, (size_t)(a - base) }; + g_wrapped[lo] = b; + return { b, (size_t)(a - lo) }; } std::pair, size_t> bufForPtrOff(const void* p, size_t bytes) { @@ -338,6 +351,26 @@ void cactus_metal_session_flush() { g_shared[reinterpret_cast(c)] = b; return c; } + +static char* g_slab = nullptr; +static size_t g_slab_used = 0, g_slab_cap = 0; +void* cactus_metal_alloc_pooled(size_t bytes) { + if (!ctx().ok) return nullptr; + if (bytes > (4u << 20)) return cactus_metal_alloc_shared(bytes); + size_t need = (bytes + 255) & ~size_t(255); + if (!g_slab || g_slab_used + need > g_slab_cap) { + size_t cap = 32u << 20; + id b = [ctx().dev newBufferWithLength:cap options:MTLResourceStorageModeShared]; + if (!b) return cactus_metal_alloc_shared(bytes); + g_slab = (char*)[b contents]; + g_slab_used = 0; + g_slab_cap = cap; + g_shared[reinterpret_cast(g_slab)] = b; + } + void* p = g_slab + g_slab_used; + g_slab_used += need; + return p; +} void cactus_metal_free_shared(void* contents) { auto it = g_shared.find(reinterpret_cast(contents)); if (it != g_shared.end()) { g_pending.push_back(it->second); g_shared.erase(it); } diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index fcd068603..d6ae6772a 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -23,6 +23,7 @@ void cactus_metal_session_flush() {} void cactus_metal_invalidate_host_wraps() {} void cactus_metal_session_end() {} void* cactus_metal_alloc_shared(size_t) { return nullptr; } +void* cactus_metal_alloc_pooled(size_t) { return nullptr; } void cactus_metal_free_shared(void*) {} bool cactus_metal_encode_copy(void*, const void*, size_t) { return false; } bool cactus_metal_encode_binary(int, void*, const void*, const void*, size_t) { return false; } diff --git a/python/cactus/bindings/cactus.py b/python/cactus/bindings/cactus.py index b575f85b6..5eae1feee 100644 --- a/python/cactus/bindings/cactus.py +++ b/python/cactus/bindings/cactus.py @@ -1489,6 +1489,7 @@ class Graph: CQ4 = 6 CPU = 0 NPU = 1 + METAL = 2 ACT_SILU = 0 ACT_GELU = 1 ACT_GELU_ERF = 2 From c793e479ce4f6be5d6fbd0a30fa851b66d35611a Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 20:53:56 +0200 Subject: [PATCH 15/36] Metal pipeline optimization: fix flushed-command-buffer sync hazard, batch-fuse unrolled tower subgraphs, MPS attention path - session_sync now waits on the last committed command buffer; previously flushed-but-unfinished GPU work was never awaited, corrupting CPU reads that raced it (output depended on flush cadence) - wrapHostPtr wraps whole VM regions, eliminating overlapping MTLBuffer wraps over the same memory - metal plan rule 8: CAT-of-matmuls over dense INDEX chains -> single batched gemm (fp16 or fp32 lhs, absorbed casts) - metal plan rule 9: CAT-of-depthwise-conv1d slices -> single depthwise conv kernel - metal plan rule 10: CAT-of-transposes at fixed source stride -> single strided copy - activation arena restricted to attention-clustered (decode) plans - MPS-based dense attention for large non-causal sequences with optional boolean mask folded via beta=1 - vectorized strided copy for contiguous inner rows; scalar precision casts execute on CPU when inputs are host-resident - gemma-3n audio tower 414ms -> 192ms, vision tower 440ms -> 376ms, outputs match CPU; llm/vlm/stt suites pass; decode tps unchanged Signed-off-by: jakmro --- cactus-graph/src/execute.cpp | 4 +- cactus-graph/src/metal_plan.cpp | 316 +++++++++++++++++++++- cactus-kernels/metal_backend.h | 4 + cactus-kernels/src/cactus_kernels.metal | 70 +++++ cactus-kernels/src/metal_backend.mm | 156 ++++++++++- cactus-kernels/src/metal_backend_stub.cpp | 2 + 6 files changed, 542 insertions(+), 10 deletions(-) diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 1c8680d38..5d086b331 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -1194,7 +1194,9 @@ void CactusGraph::execute(const std::string& profile_file) { if (it != node_index_map_.end()) flip(nodes_[it->second]->output_buffer); } } - bool encoded = try_encode_metal(*node, nodes_, node_index_map_); + bool force_cpu = ot == OpType::PRECISION_CAST + && node->output_buffer.total_size <= 8 && !input_metal_live(*node); + bool encoded = !force_cpu && try_encode_metal(*node, nodes_, node_index_map_); for (auto& fp : flipped) fp.first->precision = fp.second; if (!encoded && rplan && rplan[i] == 2) { metal_retype_disabled_ = true; diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index 82ea1ca7e..309a97d34 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -176,6 +176,42 @@ MetalFusePlan* cactus_metal_plan_build( return true; }; + auto alias_base = [&](long i, long* off_out) -> long { + long off = 0; + while (i >= 0) { + const GraphNode& nd = *nodes[(size_t)i]; + if (passthrough(nd) && !nd.input_ids.empty()) { + i = idxof(nd.input_ids[0]); + continue; + } + if (nd.op_type == OpType::INDEX && nd.params.axis == 0 && !nd.input_ids.empty()) { + long blk = 1; + for (size_t d : nd.output_buffer.shape) blk *= (long)d; + off += (long)nd.params.index_value * blk; + i = idxof(nd.input_ids[0]); + continue; + } + if (nd.op_type == OpType::SLICE && !nd.input_ids.empty()) { + long j = idxof(nd.input_ids[0]); + if (j < 0) break; + const auto& ish = nodes[(size_t)j]->output_buffer.shape; + size_t ax = (size_t)nd.params.axis; + if (ax >= ish.size()) break; + size_t outer = 1; + for (size_t d = 0; d < ax; ++d) outer *= ish[d]; + if (outer != 1) break; + long inner = 1; + for (size_t d = ax + 1; d < ish.size(); ++d) inner *= (long)ish[d]; + off += (long)nd.params.slice_start * inner; + i = j; + continue; + } + break; + } + *off_out = off; + return i; + }; + auto add_cluster = [&](MetalCluster c, size_t anchor, const std::vector& cover) { int32_t cid = (int32_t)plan->clusters.size(); plan->clusters.push_back(c); @@ -419,6 +455,228 @@ MetalFusePlan* cactus_metal_plan_build( } } + if (nd.op_type == OpType::CAT && nd.input_ids.size() >= 4 && !nd.output_buffer.shape.empty()) { + size_t cnt = nd.input_ids.size(); + bool blocky = true; + { + size_t ax = (size_t)nd.params.axis; + const auto& osh = nd.output_buffer.shape; + if (ax >= osh.size()) blocky = false; + for (size_t d = 0; blocky && d < ax; ++d) if (osh[d] != 1) blocky = false; + long fi = idxof(nd.input_ids[0]); + if (blocky && fi >= 0) { + const auto& ish = nodes[(size_t)fi]->output_buffer.shape; + size_t ip = 1, op2 = 1; + for (size_t d : ish) ip *= d; + for (size_t d : osh) op2 *= d; + if (ip * cnt != op2) blocky = false; + } else blocky = false; + } + if (!blocky) { } + else { + long m0 = -1; + { + long t = up_in(i, 0); + while (t >= 0 && nodes[(size_t)t]->op_type == OpType::PRECISION_CAST + && !nodes[(size_t)t]->input_ids.empty()) + t = up(idxof(nodes[(size_t)t]->input_ids[0])); + m0 = t; + } + if (m0 >= 0 && nodes[(size_t)m0]->op_type == OpType::MATMUL + && nodes[(size_t)m0]->output_buffer.shape.size() == 2 + && nodes[(size_t)m0]->output_buffer.precision == Precision::FP16) { + size_t M = nodes[(size_t)m0]->output_buffer.shape[0]; + size_t N = nodes[(size_t)m0]->output_buffer.shape[1]; + long ka = idxof(nodes[(size_t)m0]->input_ids[0]); + size_t K = ka >= 0 ? nodes[(size_t)ka]->output_buffer.shape.back() : 0; + bool f32out = nd.output_buffer.precision == Precision::FP32; + long parentA = -1, parentB = -1, offA0 = 0, offB0 = 0; + bool ok = K > 0; + bool f32a = false; + std::vector cover; + for (size_t j = 0; j < cnt && ok; ++j) { + long t = idxof(nd.input_ids[j]); + std::vector chain; + while (t >= 0 && (passthrough(*nodes[(size_t)t]) + || (nodes[(size_t)t]->op_type == OpType::PRECISION_CAST))) { + if (cons[(size_t)t].size() != 1) { t = -1; break; } + chain.push_back((size_t)t); + t = idxof(nodes[(size_t)t]->input_ids[0]); + } + if (t < 0 || nodes[(size_t)t]->op_type != OpType::MATMUL + || plan->action[(size_t)t] != -1 + || cons[(size_t)t].size() != 1 + || nodes[(size_t)t]->output_buffer.shape.size() != 2 + || nodes[(size_t)t]->output_buffer.shape[0] != M + || nodes[(size_t)t]->output_buffer.shape[1] != N + || nodes[(size_t)t]->params.pretransposed_rhs) { ok = false; break; } + long offA, offB; + long lin = idxof(nodes[(size_t)t]->input_ids[0]); + bool jf32 = false; + long lcast = -1; + if (lin >= 0 && nodes[(size_t)lin]->op_type == OpType::PRECISION_CAST + && nodes[(size_t)lin]->output_buffer.precision == Precision::FP16 + && cons[(size_t)lin].size() == 1 + && !nodes[(size_t)lin]->input_ids.empty()) { + long ci = idxof(nodes[(size_t)lin]->input_ids[0]); + if (ci >= 0 && nodes[(size_t)ci]->output_buffer.precision == Precision::FP32) { + jf32 = true; lcast = lin; lin = ci; + } + } + long pa = alias_base(lin, &offA); + long pb = alias_base(idxof(nodes[(size_t)t]->input_ids[1]), &offB); + Precision wantA = jf32 ? Precision::FP32 : Precision::FP16; + if (pa < 0 || pb < 0 + || nodes[(size_t)pa]->output_buffer.precision != wantA + || nodes[(size_t)pb]->output_buffer.precision != Precision::FP16) { ok = false; break; } + if (j == 0) { parentA = pa; parentB = pb; offA0 = offA; offB0 = offB; f32a = jf32; } + else if (jf32 != f32a) { ok = false; break; } + else if (pa != parentA || pb != parentB + || offA != offA0 + (long)(j * M * K) + || offB != offB0 + (long)(j * K * N)) { ok = false; break; } + cover.push_back((size_t)t); + if (lcast >= 0) cover.push_back((size_t)lcast); + for (size_t cnode : chain) cover.push_back(cnode); + } + if (ok) { + MetalCluster c; + c.rule = 8; + c.a0 = (size_t)parentA; c.a1 = (size_t)parentB; + c.a2 = (size_t)offA0; c.a3 = (size_t)offB0; + c.u0 = (uint32_t)cnt; c.u1 = (uint32_t)M; + c.b0 = K; c.b1 = N; c.b2 = f32out ? 1 : 0; + c.b3 = f32a ? 1 : 0; + c.b4 = i; + add_cluster(c, i, cover); + continue; + } + } + } + bool samesz = true; + { + long fi = idxof(nd.input_ids[0]); + if (fi < 0) samesz = false; + else { + size_t ip = nodes[(size_t)fi]->output_buffer.total_size; + samesz = ip * cnt == nd.output_buffer.total_size; + } + } + long tp0 = samesz ? up_in(i, 0) : -1; + if (tp0 >= 0 && nodes[(size_t)tp0]->op_type == OpType::TRANSPOSE + && nodes[(size_t)tp0]->output_buffer.precision == Precision::FP16 + && nd.output_buffer.precision == Precision::FP16 + && nodes[(size_t)tp0]->output_buffer.shape.size() + 1 <= 8 + && !is_noop_transpose(*nodes[(size_t)tp0])) { + const auto& perm0 = nodes[(size_t)tp0]->params.permutation; + const auto& tsh0 = nodes[(size_t)tp0]->output_buffer.shape; + long parentP = -1, offP0 = 0, strideP = -1; + size_t blk = nodes[(size_t)tp0]->output_buffer.total_size; + bool ok = !perm0.empty(); + std::vector cover; + for (size_t j = 0; j < cnt && ok; ++j) { + long t = idxof(nd.input_ids[j]); + std::vector chain; + while (t >= 0 && passthrough(*nodes[(size_t)t])) { + if (cons[(size_t)t].size() != 1) { t = -1; break; } + chain.push_back((size_t)t); + t = idxof(nodes[(size_t)t]->input_ids[0]); + } + if (t < 0 || nodes[(size_t)t]->op_type != OpType::TRANSPOSE + || plan->action[(size_t)t] != -1 + || cons[(size_t)t].size() != 1 + || nodes[(size_t)t]->params.permutation != perm0 + || nodes[(size_t)t]->output_buffer.shape != tsh0 + || nodes[(size_t)t]->output_buffer.precision != Precision::FP16) { ok = false; break; } + long offP; + long pp = alias_base(idxof(nodes[(size_t)t]->input_ids[0]), &offP); + if (pp < 0 || nodes[(size_t)pp]->output_buffer.precision != Precision::FP16) { ok = false; break; } + long ii = idxof(nodes[(size_t)t]->input_ids[0]); + if (ii < 0 || nodes[(size_t)ii]->output_buffer.total_size != blk) { ok = false; break; } + if (j == 0) { parentP = pp; offP0 = offP; } + else if (j == 1 && pp == parentP && offP > offP0) { strideP = offP - offP0; } + else if (j == 1 || pp != parentP || offP != offP0 + (long)j * strideP) { ok = false; break; } + cover.push_back((size_t)t); + for (size_t cnode : chain) cover.push_back(cnode); + } + if (ok && (strideP < 0 || cnt < 2)) ok = false; + if (ok) { + size_t ax = (size_t)nd.params.axis; + long fi = idxof(nd.input_ids[0]); + const auto& fsh = nodes[(size_t)fi]->output_buffer.shape; + if (ax >= fsh.size()) ok = false; + else { + size_t pre = 1; + for (size_t d = 0; d < ax && d < fsh.size(); ++d) pre *= fsh[d]; + size_t fpre = 1; + const auto& csh = nd.output_buffer.shape; + for (size_t d = 0; d < ax && d < csh.size(); ++d) fpre *= csh[d]; + if (pre != fpre) ok = false; + } + } + if (ok) { + MetalCluster c; + c.rule = 10; + c.a0 = (size_t)parentP; c.a1 = (size_t)up_in(i, 0); + c.a2 = (size_t)offP0; + c.u0 = (uint32_t)cnt; + c.u1 = (uint32_t)nd.params.axis; + c.b0 = (size_t)strideP; + c.b4 = i; + add_cluster(c, i, cover); + continue; + } + } + + long c0 = up_in(i, 0); + if (!blocky) c0 = -1; + if (c0 >= 0 && nodes[(size_t)c0]->op_type == OpType::CONV1D + && nodes[(size_t)c0]->output_buffer.precision == Precision::FP16) { + const auto& osh = nodes[(size_t)c0]->output_buffer.shape; + size_t Lout = osh.empty() ? 0 : osh.back(); + size_t stride = nodes[(size_t)c0]->params.stride ? nodes[(size_t)c0]->params.stride : 1; + long parentX = -1, parentW = -1, offX0 = 0, offW0 = 0; + size_t L = 0, Kk = 0; + bool ok = Lout > 0; + std::vector cover; + for (size_t j = 0; j < cnt && ok; ++j) { + long t = up(idxof(nd.input_ids[j])); + if (t < 0 || nodes[(size_t)t]->op_type != OpType::CONV1D + || plan->action[(size_t)t] != -1 + || cons[(size_t)t].size() != 1 + || nodes[(size_t)t]->input_ids.size() != 2 + || nodes[(size_t)t]->output_buffer.shape.back() != Lout) { ok = false; break; } + long offX, offW; + long px = alias_base(idxof(nodes[(size_t)t]->input_ids[0]), &offX); + long pw = alias_base(idxof(nodes[(size_t)t]->input_ids[1]), &offW); + if (px < 0 || pw < 0 + || nodes[(size_t)px]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)pw]->output_buffer.precision != Precision::FP16) { ok = false; break; } + long xin = idxof(nodes[(size_t)t]->input_ids[0]); + long win = idxof(nodes[(size_t)t]->input_ids[1]); + size_t Lj = nodes[(size_t)xin]->output_buffer.shape.back(); + size_t Kj = nodes[(size_t)win]->output_buffer.shape.back(); + if (j == 0) { + parentX = px; parentW = pw; offX0 = offX; offW0 = offW; L = Lj; Kk = Kj; + if ((L - Kk) / stride + 1 != Lout) { ok = false; break; } + } else if (px != parentX || pw != parentW || Lj != L || Kj != Kk + || offX != offX0 + (long)(j * L) + || offW != offW0 + (long)(j * Kk)) { ok = false; break; } + cover.push_back((size_t)t); + } + if (ok) { + MetalCluster c; + c.rule = 9; + c.a0 = (size_t)parentX; c.a1 = (size_t)parentW; + c.a2 = (size_t)offX0; c.a3 = (size_t)offW0; + c.u0 = (uint32_t)cnt; c.u1 = (uint32_t)L; + c.b0 = Lout; c.b1 = Kk; c.b2 = stride; + c.b4 = i; + add_cluster(c, i, cover); + continue; + } + } + } + if (nd.op_type == OpType::ADD_CLIPPED && nd.input_ids.size() == 2) { long n0 = up_in(i, 0), n1 = up_in(i, 1); long norm = -1, res = -1; @@ -618,7 +876,9 @@ MetalFusePlan* cactus_metal_plan_build( if (plan->action[i] != -2 && nodes[i]->op_type != OpType::INPUT) plan->exec_list.push_back((uint32_t)i); - if (!plan->clusters.empty()) { + bool decode_like = false; + for (auto& cl : plan->clusters) if (cl.rule == 1) { decode_like = true; break; } + if (decode_like) { std::vector last_read(n, 0); auto base_of = [&](size_t i) -> size_t { size_t cur = i; @@ -707,6 +967,7 @@ MetalFusePlan* cactus_metal_plan_build( void cactus_metal_plan_free(MetalFusePlan* p) { delete p; } + void cactus_metal_plan_disable(MetalFusePlan* p) { if (!p) return; std::fill(p->action.begin(), p->action.end(), -1); @@ -908,6 +1169,59 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, } return cactus_graph_metal_tail(lg, V); } + case 8: { + GraphNode& anchor = *nodes[c.b4]; + const char* pa = (const char*)nodes[c.a0]->output_buffer.get_data(); + const char* pb = (const char*)nodes[c.a1]->output_buffer.get_data(); + void* out = anchor.output_buffer.get_data(); + if (!pa || !pb || !out) return false; + return cactus_metal_encode_gemm_batch(out, pa + c.a2 * (c.b3 ? 4 : 2), pb + c.a3 * 2, + c.u1, (uint32_t)c.b0, (uint32_t)c.b1, c.u0, (int)c.b2, (int)c.b3); + } + case 9: { + GraphNode& anchor = *nodes[c.b4]; + const char* px = (const char*)nodes[c.a0]->output_buffer.get_data(); + const char* pw = (const char*)nodes[c.a1]->output_buffer.get_data(); + void* out = anchor.output_buffer.get_data(); + if (!px || !pw || !out) return false; + return cactus_metal_encode_conv1d_dw(out, px + c.a2 * 2, pw + c.a3 * 2, + c.u0, c.u1, (uint32_t)c.b0, (uint32_t)c.b1, (uint32_t)c.b2); + } + case 10: { + GraphNode& anchor = *nodes[c.b4]; + GraphNode& tn = *nodes[c.a1]; + const char* pp = (const char*)nodes[c.a0]->output_buffer.get_data(); + void* out = anchor.output_buffer.get_data(); + if (!pp || !out) return false; + long ii = -1; + { + auto it = map.find(tn.input_ids[0]); + if (it == map.end()) return false; + ii = (long)it->second; + } + const auto& ish = nodes[(size_t)ii]->output_buffer.shape; + const auto& osh = tn.output_buffer.shape; + const auto& perm = tn.params.permutation; + uint32_t ndim = (uint32_t)perm.size(); + uint32_t ax = c.u1; + if (ndim == 0 || ndim + 1 > 8 || ish.size() != ndim || osh.size() != ndim || ax > ndim) return false; + size_t istr[8]; + istr[ndim - 1] = 1; + for (size_t d = ndim - 1; d > 0; --d) istr[d - 1] = istr[d] * ish[d]; + uint32_t oshape[8], sstride[8]; + for (uint32_t d = 0; d < ax; ++d) { + oshape[d] = (uint32_t)osh[d]; + sstride[d] = (uint32_t)istr[perm[d]]; + } + oshape[ax] = c.u0; sstride[ax] = (uint32_t)c.b0; + for (uint32_t d = ax; d < ndim; ++d) { + oshape[d + 1] = (uint32_t)osh[d]; + sstride[d + 1] = (uint32_t)istr[perm[d]]; + } + size_t total = (size_t)c.u0 * tn.output_buffer.total_size; + return cactus_metal_encode_strided_copy(out, pp + c.a2 * 2, oshape, sstride, ndim + 1, + (uint32_t)total, 0, nodes[c.a0]->output_buffer.byte_size - c.a2 * 2, anchor.output_buffer.byte_size); + } default: return false; } } diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index d6383c0bf..a29c60076 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -117,6 +117,10 @@ bool cactus_metal_encode_gemm_f16(void* out, const void* lhs, const void* rhs, bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, const void* v, const void* mask, uint32_t B, uint32_t T, uint32_t S, uint32_t HQ, uint32_t HKV, uint32_t D, uint32_t DV, float scale, uint32_t causal, uint32_t pos_off, uint32_t window, float logit_cap, uint32_t mask_mode); +bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, + uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a); +bool cactus_metal_encode_conv1d_dw(void* out, const void* x, const void* w, + uint32_t C, uint32_t L, uint32_t Lout, uint32_t K, uint32_t stride); bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* oshape, const uint32_t* sstride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes); diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index fb40d22f6..cbee12de0 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -1091,6 +1091,20 @@ kernel void strided_copy_f16(device const half* in [[buffer(0)]], device half* o out[i]=in[src]; } +kernel void strided_copy_rows_f16(device const half* in [[buffer(0)]], device half* out [[buffer(1)]], + constant uint* oshape [[buffer(2)]], constant uint* sstride [[buffer(3)]], + constant uint& ndim [[buffer(4)]], constant uint& rows [[buffer(5)]], constant uint& base [[buffer(6)]], + constant uint& inner4 [[buffer(7)]], + uint2 gid [[thread_position_in_grid]]) { + uint v = gid.x, row = gid.y; + if (v >= inner4 || row >= rows) return; + uint rem = row, src = base; + for (int d = int(ndim) - 1; d >= 0; --d) { uint c = rem % oshape[d]; rem /= oshape[d]; src += c * sstride[d]; } + device const half4* s4 = (device const half4*)(in + src); + device half4* d4 = (device half4*)(out + (size_t)row * inner4 * 4); + d4[v] = s4[v]; +} + kernel void kv_append_i8(device const half* src [[buffer(0)]], device char* int8base [[buffer(1)]], device float* scalebase [[buffer(2)]], constant uint& kv_heads [[buffer(3)]], constant uint& hdim [[buffer(4)]], constant uint& current_len [[buffer(5)]], constant uint& group_size [[buffer(6)]], @@ -1509,6 +1523,62 @@ kernel void attn_f16_d64(device const half* q [[buffer(0)]], device const half* } } +kernel void attn_maskfill_f16(device const half* mask [[buffer(0)]], device half* out [[buffer(1)]], + constant uint& ts [[buffer(2)]], + uint2 gid [[thread_position_in_grid]]) { + uint i = gid.x, h = gid.y; + if (i >= ts) return; + out[(size_t)h*ts + i] = (mask[i] == (half)0.0f) ? (half)(-30000.0f) : (half)0.0f; +} + +kernel void gemm_batch_f32a(device const float* a [[buffer(0)]], device const half* b [[buffer(1)]], + device uchar* y [[buffer(2)]], + constant uint& M [[buffer(3)]], constant uint& K [[buffer(4)]], + constant uint& N [[buffer(5)]], constant uint& f32out [[buffer(6)]], + uint2 gid [[thread_position_in_grid]]) { + uint mn = gid.x, bidx = gid.y; + if (mn >= M*N) return; + uint m = mn / N, nn = mn % N; + device const float* ar = a + (size_t)bidx*M*K + (size_t)m*K; + device const half* br = b + (size_t)bidx*K*N + nn; + float acc = 0; + for (uint k = 0; k < K; ++k) acc = fma((float)((half)ar[k]), (float)br[(size_t)k*N], acc); + size_t oi = (size_t)bidx*M*N + mn; + if (f32out != 0u) ((device float*)y)[oi] = acc; + else ((device half*)y)[oi] = (half)acc; +} + +kernel void gemm_batch_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], + device uchar* y [[buffer(2)]], + constant uint& M [[buffer(3)]], constant uint& K [[buffer(4)]], + constant uint& N [[buffer(5)]], constant uint& f32out [[buffer(6)]], + uint2 gid [[thread_position_in_grid]]) { + uint mn = gid.x, bidx = gid.y; + if (mn >= M*N) return; + uint m = mn / N, nn = mn % N; + device const half* ar = a + (size_t)bidx*M*K + (size_t)m*K; + device const half* br = b + (size_t)bidx*K*N + nn; + float acc = 0; + for (uint k = 0; k < K; ++k) acc = fma((float)ar[k], (float)br[(size_t)k*N], acc); + size_t oi = (size_t)bidx*M*N + mn; + if (f32out != 0u) ((device float*)y)[oi] = acc; + else ((device half*)y)[oi] = (half)acc; +} + +kernel void conv1d_dw_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device half* y [[buffer(2)]], + constant uint& L [[buffer(3)]], constant uint& Lout [[buffer(4)]], + constant uint& Kk [[buffer(5)]], constant uint& stride [[buffer(6)]], + uint2 gid [[thread_position_in_grid]]) { + uint l = gid.x, c = gid.y; + if (l >= Lout) return; + device const half* xc = x + (size_t)c*L + (size_t)l*stride; + device const half* wc = w + (size_t)c*Kk; + float acc = 0; + for (uint k = 0; k < Kk; ++k) acc = fma((float)xc[k], (float)wc[k], acc); + y[(size_t)c*Lout + l] = (half)acc; +} + kernel void bcast_binary_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], device half* out [[buffer(2)]], constant uint* oshape [[buffer(3)]], constant uint* astride [[buffer(4)]], constant uint* bstride [[buffer(5)]], diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index fc2945226..e636cdff2 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -6,6 +6,7 @@ #import #import +#include #import #include #include @@ -49,6 +50,7 @@ id psoBinF32=nil, psoScaF32=nil, psoUnaF32=nil; id psoClampF16=nil, psoClampF32=nil, psoGlu=nil; id psoLayerNorm=nil, psoSoftmaxR=nil, psoConv1dK3=nil, psoAttnDense=nil, psoAttnD64=nil; + id psoGemmBatch=nil, psoGemmBatchF32A=nil, psoConvDw=nil, psoStridedRows=nil, psoMaskFill=nil; id dummy=nil; bool ok = false; @@ -94,6 +96,8 @@ psoClampF16=pso("clamp_f16"); psoClampF32=pso("clamp_f32"); psoGlu=pso("glu_f16"); psoLayerNorm=pso("layer_norm_f16"); psoSoftmaxR=pso("softmax_rows_f16"); psoConv1dK3=pso("conv1d_k3_f16"); psoAttnDense=pso("attn_f16"); psoAttnD64=pso("attn_f16_d64"); + psoGemmBatch=pso("gemm_batch_f16"); psoGemmBatchF32A=pso("gemm_batch_f32a"); psoConvDw=pso("conv1d_dw_f16"); + psoStridedRows=pso("strided_copy_rows_f16"); psoMaskFill=pso("attn_maskfill_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; @@ -261,6 +265,22 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, } size_t wraplen = (need + 16383u) & ~(size_t)16383u; uintptr_t lo = base, hi = base + wraplen; + { + mach_vm_address_t raddr = a; + mach_vm_size_t rsize = 0; + vm_region_basic_info_data_64_t info; + mach_msg_type_number_t cnt = VM_REGION_BASIC_INFO_COUNT_64; + mach_port_t obj = MACH_PORT_NULL; + if (mach_vm_region(mach_task_self(), &raddr, &rsize, VM_REGION_BASIC_INFO_64, + (vm_region_info_t)&info, &cnt, &obj) == KERN_SUCCESS + && raddr <= base && raddr + rsize >= hi + && (info.protection & VM_PROT_READ) && (info.protection & VM_PROT_WRITE) + && rsize <= (512u << 20)) { + lo = (uintptr_t)raddr; + hi = (uintptr_t)(raddr + rsize); + } + if (obj != MACH_PORT_NULL) mach_port_deallocate(mach_task_self(), obj); + } auto ov = g_wrapped.lower_bound(lo); if (ov != g_wrapped.begin()) { auto prev = std::prev(ov); @@ -312,12 +332,18 @@ inline void setBufAt(const void* p, size_t bytes, int idx) { bool cactus_metal_active_mode() { return ctx().ok && g_active; } void cactus_metal_session_begin() {} +static id g_last_cmd = nil; void cactus_metal_session_sync() { if (g_enc) { [g_enc endEncoding]; g_enc = nil; } if (g_cmd) { - [g_cmd commit]; [g_cmd waitUntilCompleted]; + [g_cmd commit]; + g_last_cmd = g_cmd; g_cmd = nil; } + if (g_last_cmd) { + [g_last_cmd waitUntilCompleted]; + g_last_cmd = nil; + } for (id b : g_pending) g_free[(size_t)b.length].push_back(b); g_pending.clear(); } @@ -336,7 +362,7 @@ void cactus_metal_invalidate_host_wraps() { void cactus_metal_session_flush() { if (g_enc) { [g_enc endEncoding]; g_enc = nil; } - if (g_cmd) { [g_cmd commit]; g_cmd = nil; } + if (g_cmd) { [g_cmd commit]; g_last_cmd = g_cmd; g_cmd = nil; } } @@ -1314,17 +1340,26 @@ bool cactus_metal_encode_conv1d_k3(void* out, const void* x, const void* w, int [g_enc dispatchThreads:MTLSizeMake(Lout,Cout,1) threadsPerThreadgroup:MTLSizeMake(64,4,1)]; return true; } -static MPSMatrixMultiplication* mps_gemm(uint32_t M, uint32_t N, uint32_t K, bool tr) { - static std::map, MPSMatrixMultiplication*> cache; - auto key = std::make_tuple(M,N,K,tr); +static MPSMatrixMultiplication* mps_gemm_ab(uint32_t M, uint32_t N, uint32_t K, bool tr, float alpha, float beta) { + static std::map, MPSMatrixMultiplication*> cache; + uint32_t ab, bb; + std::memcpy(&ab, &alpha, 4); + std::memcpy(&bb, &beta, 4); + auto key = std::make_tuple(M,N,K,tr,((uint64_t)ab<<32)|bb); auto it = cache.find(key); if (it != cache.end()) return it->second; MPSMatrixMultiplication* mm = [[MPSMatrixMultiplication alloc] initWithDevice:ctx().dev transposeLeft:false transposeRight:tr - resultRows:M resultColumns:N interiorColumns:K alpha:1.0 beta:0.0]; + resultRows:M resultColumns:N interiorColumns:K alpha:alpha beta:beta]; cache[key] = mm; return mm; } +static MPSMatrixMultiplication* mps_gemm_a(uint32_t M, uint32_t N, uint32_t K, bool tr, float alpha) { + return mps_gemm_ab(M, N, K, tr, alpha, 0.0f); +} +static MPSMatrixMultiplication* mps_gemm(uint32_t M, uint32_t N, uint32_t K, bool tr) { + return mps_gemm_a(M, N, K, tr, 1.0f); +} bool cactus_metal_encode_gemm_f16(void* out, const void* lhs, const void* rhs, uint32_t M, uint32_t K, uint32_t N, int pretransposed) { @@ -1333,13 +1368,14 @@ bool cactus_metal_encode_gemm_f16(void* out, const void* lhs, const void* rhs, auto a = bufForPtrOff(lhs, (size_t)M*K*2); auto b = bufForPtrOff(rhs, (size_t)N*K*2); auto c = bufForPtrOff(out, (size_t)M*N*2); + int use_tr = pretransposed; if (mps_ok && (a.second % 16)==0 && (b.second % 16)==0 && (c.second % 16)==0 && (K % 8)==0 && (N % 8)==0) { if (g_enc) { [g_enc endEncoding]; g_enc = nil; } if (!g_cmd) g_cmd = [ctx().queue commandBuffer]; MPSMatrixDescriptor* da = [MPSMatrixDescriptor matrixDescriptorWithRows:M columns:K rowBytes:(size_t)K*2 dataType:MPSDataTypeFloat16]; - MPSMatrixDescriptor* db = pretransposed + MPSMatrixDescriptor* db = use_tr ? [MPSMatrixDescriptor matrixDescriptorWithRows:N columns:K rowBytes:(size_t)K*2 dataType:MPSDataTypeFloat16] : [MPSMatrixDescriptor matrixDescriptorWithRows:K columns:N rowBytes:(size_t)N*2 dataType:MPSDataTypeFloat16]; MPSMatrixDescriptor* dc = [MPSMatrixDescriptor matrixDescriptorWithRows:M columns:N @@ -1356,7 +1392,7 @@ bool cactus_metal_encode_gemm_f16(void* out, const void* lhs, const void* rhs, MPSMatrix* ma = mat(a.first, a.second, da); MPSMatrix* mb = mat(b.first, b.second, db); MPSMatrix* mc = mat(c.first, c.second, dc); - [mps_gemm(M, N, K, pretransposed != 0) encodeToCommandBuffer:g_cmd + [mps_gemm(M, N, K, use_tr != 0) encodeToCommandBuffer:g_cmd leftMatrix:ma rightMatrix:mb resultMatrix:mc]; return true; } @@ -1388,6 +1424,73 @@ bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, float scale, uint32_t causal, uint32_t pos_off, uint32_t window, float logit_cap, uint32_t mask_mode) { if (!ctx().ok || !ctx().psoAttnDense) return false; if (D > 128 || DV > 128 || (D % 4) != 0 || (DV % 4) != 0 || HKV == 0 || HQ % HKV != 0) return false; + static const bool mps_attn_ok = MPSSupportsMTLDevice(ctx().dev); + bool mask_ok = mask_mode == 0 || (mask_mode == 2 && mask && ctx().psoMaskFill); + if (mps_attn_ok && B == 1 && HQ == HKV && !causal && mask_ok && window == 0 + && logit_cap == 0.0f && S >= 512 && (S % 8) == 0 && (D % 8) == 0 && (DV % 8) == 0 + && ctx().psoSoftmaxR) { + auto qb = bufForPtrOff(q, (size_t)T*HQ*D*2); + auto kb = bufForPtrOff(k, (size_t)S*HKV*D*2); + auto vb = bufForPtrOff(v, (size_t)S*HKV*DV*2); + auto ob = bufForPtrOff(out, (size_t)T*HQ*DV*2); + bool aligned = (qb.second % 16)==0 && (kb.second % 16)==0 && (vb.second % 16)==0 && (ob.second % 16)==0 + && ((D*2) % 16)==0 && ((DV*2) % 16)==0; + if (aligned) { + static id sbuf = nil; + size_t need = (size_t)HQ*T*S*2; + if (!sbuf || sbuf.length < need) + sbuf = [ctx().dev newBufferWithLength:need options:MTLResourceStorageModeShared]; + if (sbuf) { + float beta = 0.0f; + if (mask_mode == 2) { + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoMaskFill]; + setBufAt(mask, (size_t)T*S*2, 0); + [g_enc setBuffer:sbuf offset:0 atIndex:1]; + uint32_t ts = T*S; + [g_enc setBytes:&ts length:4 atIndex:2]; + [g_enc dispatchThreads:MTLSizeMake(ts, HQ, 1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + beta = 1.0f; + } + if (g_enc) { [g_enc endEncoding]; g_enc = nil; } + if (!g_cmd) g_cmd = [ctx().queue commandBuffer]; + MPSMatrixDescriptor* dq = [MPSMatrixDescriptor matrixDescriptorWithRows:T columns:D + rowBytes:(size_t)HQ*D*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* dk = [MPSMatrixDescriptor matrixDescriptorWithRows:S columns:D + rowBytes:(size_t)HKV*D*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* ds = [MPSMatrixDescriptor matrixDescriptorWithRows:T columns:S + rowBytes:(size_t)S*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* dv = [MPSMatrixDescriptor matrixDescriptorWithRows:S columns:DV + rowBytes:(size_t)HKV*DV*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* do2 = [MPSMatrixDescriptor matrixDescriptorWithRows:T columns:DV + rowBytes:(size_t)HQ*DV*2 dataType:MPSDataTypeFloat16]; + MPSMatrixMultiplication* mqk = mps_gemm_ab(T, S, D, true, scale, beta); + for (uint32_t h = 0; h < HQ; ++h) { + MPSMatrix* mq = [[MPSMatrix alloc] initWithBuffer:qb.first offset:qb.second + (size_t)h*D*2 descriptor:dq]; + MPSMatrix* mk = [[MPSMatrix alloc] initWithBuffer:kb.first offset:kb.second + (size_t)h*D*2 descriptor:dk]; + MPSMatrix* msc = [[MPSMatrix alloc] initWithBuffer:sbuf offset:(size_t)h*T*S*2 descriptor:ds]; + [mqk encodeToCommandBuffer:g_cmd leftMatrix:mq rightMatrix:mk resultMatrix:msc]; + } + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoSoftmaxR]; + [g_enc setBuffer:sbuf offset:0 atIndex:0]; + [g_enc setBuffer:sbuf offset:0 atIndex:1]; + uint32_t cols = S; + [g_enc setBytes:&cols length:4 atIndex:2]; + [g_enc setThreadgroupMemoryLength:256*4 atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((size_t)HQ*T,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + [g_enc endEncoding]; g_enc = nil; + MPSMatrixMultiplication* mpv = mps_gemm(T, DV, S, false); + for (uint32_t h = 0; h < HQ; ++h) { + MPSMatrix* msc = [[MPSMatrix alloc] initWithBuffer:sbuf offset:(size_t)h*T*S*2 descriptor:ds]; + MPSMatrix* mv = [[MPSMatrix alloc] initWithBuffer:vb.first offset:vb.second + (size_t)h*DV*2 descriptor:dv]; + MPSMatrix* mo = [[MPSMatrix alloc] initWithBuffer:ob.first offset:ob.second + (size_t)h*DV*2 descriptor:do2]; + [mpv encodeToCommandBuffer:g_cmd leftMatrix:msc rightMatrix:mv resultMatrix:mo]; + } + return true; + } + } + } ensureEncoder(); struct { uint32_t T,S,HQ,HKV,D,DV; float scale; uint32_t causal,pos_off,window; float logit_cap; uint32_t mask_mode; } U = { T,S,HQ,HKV,D,DV, scale, causal,pos_off,window, logit_cap, mask_mode }; @@ -1404,10 +1507,47 @@ bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, return true; } +bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, + uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a) { + if (!ctx().ok || !ctx().psoGemmBatch || !ctx().psoGemmBatchF32A) return false; + ensureEncoder(); + uint32_t fo = f32out ? 1u : 0u; + [g_enc setComputePipelineState:(f32a ? ctx().psoGemmBatchF32A : ctx().psoGemmBatch)]; + setBufAt(a, (size_t)batch*M*K*(f32a?4:2), 0); setBufAt(b, (size_t)batch*K*N*2, 1); + setBufAt(out, (size_t)batch*M*N*(f32out?4:2), 2); + [g_enc setBytes:&M length:4 atIndex:3]; [g_enc setBytes:&K length:4 atIndex:4]; + [g_enc setBytes:&N length:4 atIndex:5]; [g_enc setBytes:&fo length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake((size_t)M*N, batch, 1) threadsPerThreadgroup:MTLSizeMake(64,4,1)]; + return true; +} +bool cactus_metal_encode_conv1d_dw(void* out, const void* x, const void* w, + uint32_t C, uint32_t L, uint32_t Lout, uint32_t K, uint32_t stride) { + if (!ctx().ok || !ctx().psoConvDw) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoConvDw]; + setBufAt(x, (size_t)C*L*2, 0); setBufAt(w, (size_t)C*K*2, 1); setBufAt(out, (size_t)C*Lout*2, 2); + [g_enc setBytes:&L length:4 atIndex:3]; [g_enc setBytes:&Lout length:4 atIndex:4]; + [g_enc setBytes:&K length:4 atIndex:5]; [g_enc setBytes:&stride length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(Lout, C, 1) threadsPerThreadgroup:MTLSizeMake(64,4,1)]; + return true; +} bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* oshape, const uint32_t* sstride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes) { if (!ctx().ok || ndim == 0 || ndim > 8) return false; ensureEncoder(); + if (ndim >= 2 && sstride[ndim-1] == 1 && (oshape[ndim-1] % 4) == 0 && ctx().psoStridedRows) { + uint32_t inner = oshape[ndim-1]; + uint32_t inner4 = inner / 4; + uint32_t nd2 = ndim - 1; + uint32_t rows = total / inner; + [g_enc setComputePipelineState:ctx().psoStridedRows]; + setBufAt(in, in_bytes, 0); setBufAt(out, out_bytes, 1); + [g_enc setBytes:oshape length:nd2*4 atIndex:2]; [g_enc setBytes:sstride length:nd2*4 atIndex:3]; + [g_enc setBytes:&nd2 length:4 atIndex:4]; [g_enc setBytes:&rows length:4 atIndex:5]; + [g_enc setBytes:&base length:4 atIndex:6]; [g_enc setBytes:&inner4 length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake(inner4, rows, 1) threadsPerThreadgroup:MTLSizeMake(std::min(inner4, 64u), 4, 1)]; + return true; + } [g_enc setComputePipelineState:ctx().psoStrided]; setBufAt(in, in_bytes, 0); setBufAt(out, out_bytes, 1); [g_enc setBytes:oshape length:ndim*4 atIndex:2]; [g_enc setBytes:sstride length:ndim*4 atIndex:3]; diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index d6ae6772a..5e62e391b 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -74,6 +74,8 @@ bool cactus_metal_encode_gemm_f16(void*, const void*, const void*, uint32_t, uin bool cactus_metal_encode_attention_f16(void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, uint32_t, uint32_t, float, uint32_t) { return false; } +bool cactus_metal_encode_gemm_batch(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, int, int) { return false; } +bool cactus_metal_encode_conv1d_dw(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_strided_copy(void*, const void*, const uint32_t*, const uint32_t*, uint32_t, uint32_t, uint32_t, size_t, size_t) { return false; } bool cactus_metal_encode_bcast_binary(int, void*, const void*, const void*, const uint32_t*, From 1e77c18984547f78dfa47083205c935e583a2e37 Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 21:34:21 +0200 Subject: [PATCH 16/36] Metal encodes for the full elementwise, reduction, conv, norm, rope, and gather op set - scalar/unary/binary Metal kernels extended with exp, sqrt, cos, sin, log, abs, pow, not-equal, leaky-relu, gelu-erf, and sigmoid codes matching CPU semantics - new kernels: axis reductions (sum/mean/variance/min/max), cumsum, two-input concat, fp32-index gather, full rope (NeoX and GPT-J), maxpool1d, bilinear interpolation, generic conv1d, depthwise NLC conv1d (causal and same-k9), conv2d (dense/depthwise/pointwise), batchnorm, groupnorm, row bias add - conv1d_pointwise routed through the fp16 gemm path with fused bias pass - retype pass no longer retypes terminal nodes; their buffers are externally visible - padded cache snapshot/rollback made ring-aware for Metal-resident sliding-window caches, matching the kv_append_ring slot mapping; fixes batch-decode rollback corruption past the attention window - new test_metal_parity suite runs every newly covered op on CPU and Metal and compares outputs Signed-off-by: jakmro Signed-off-by: jakmro --- cactus-graph/src/execute.cpp | 220 ++++++++++++++- cactus-graph/src/ops_cache.cpp | 45 +++ cactus-graph/tests/test_metal_parity.cpp | 317 +++++++++++++++++++++ cactus-kernels/metal_backend.h | 31 ++ cactus-kernels/src/cactus_kernels.metal | 328 +++++++++++++++++++++- cactus-kernels/src/metal_backend.mm | 192 +++++++++++++ cactus-kernels/src/metal_backend_stub.cpp | 13 + 7 files changed, 1131 insertions(+), 15 deletions(-) create mode 100644 cactus-graph/tests/test_metal_parity.cpp diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 5d086b331..d3abf88af 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -489,11 +489,12 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n (uint32_t)rows, (uint32_t)K, (uint32_t)N, node.params.pretransposed_rhs ? 1 : 0); } case OpType::ADD: case OpType::ADD_CLIPPED: case OpType::SUBTRACT: - case OpType::MULTIPLY: case OpType::DIVIDE: { + case OpType::MULTIPLY: case OpType::DIVIDE: case OpType::NOT_EQUAL: { const auto& a = get_input(node, 0, nodes, map); const auto& b = get_input(node, 1, nodes, map); int code = node.op_type==OpType::ADD?0: node.op_type==OpType::ADD_CLIPPED?1: - node.op_type==OpType::SUBTRACT?2: node.op_type==OpType::MULTIPLY?3:4; + node.op_type==OpType::SUBTRACT?2: node.op_type==OpType::MULTIPLY?3: + node.op_type==OpType::DIVIDE?4:5; bool f32 = a.precision == Precision::FP32 && b.precision == Precision::FP32 && out.precision == Precision::FP32; if (f32 && a.total_size == out.total_size && b.total_size == out.total_size) @@ -566,23 +567,227 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n return true; } case OpType::SCALAR_ADD: case OpType::SCALAR_SUBTRACT: - case OpType::SCALAR_MULTIPLY: case OpType::SCALAR_DIVIDE: { + case OpType::SCALAR_MULTIPLY: case OpType::SCALAR_DIVIDE: + case OpType::SCALAR_EXP: case OpType::SCALAR_SQRT: case OpType::SCALAR_COS: + case OpType::SCALAR_SIN: case OpType::SCALAR_LOG: case OpType::ABS: + case OpType::POW: case OpType::SCALAR_NOT_EQUAL: case OpType::LEAKY_RELU: { const auto& in = get_input(node, 0, nodes, map); - int code = node.op_type==OpType::SCALAR_ADD?0: node.op_type==OpType::SCALAR_SUBTRACT?1: - node.op_type==OpType::SCALAR_MULTIPLY?2:3; + int code; + switch (node.op_type) { + case OpType::SCALAR_ADD: code = 0; break; + case OpType::SCALAR_SUBTRACT: code = 1; break; + case OpType::SCALAR_MULTIPLY: code = 2; break; + case OpType::SCALAR_DIVIDE: code = 3; break; + case OpType::SCALAR_EXP: code = 4; break; + case OpType::SCALAR_SQRT: code = 5; break; + case OpType::SCALAR_COS: code = 6; break; + case OpType::SCALAR_SIN: code = 7; break; + case OpType::SCALAR_LOG: code = 8; break; + case OpType::ABS: code = 9; break; + case OpType::POW: code = 10; break; + case OpType::SCALAR_NOT_EQUAL: code = 11; break; + default: code = 12; break; + } if (in.precision == Precision::FP32 && out.precision == Precision::FP32) return cactus_metal_encode_scalar_f32(code, out.get_data(), in.get_data(), out.total_size, node.params.scalar); if (!fp16(in) || !fp16(out)) return false; return cactus_metal_encode_scalar(code, out.get_data(), in.get_data(), out.total_size, node.params.scalar); } - case OpType::GELU: case OpType::TANH: case OpType::SILU: case OpType::RELU: { + case OpType::GELU: case OpType::TANH: case OpType::SILU: case OpType::RELU: + case OpType::GELU_ERF: case OpType::SIGMOID: { const auto& in = get_input(node, 0, nodes, map); - int code = node.op_type==OpType::GELU?0: node.op_type==OpType::TANH?1: node.op_type==OpType::SILU?2:3; + int code = node.op_type==OpType::GELU?0: node.op_type==OpType::TANH?1: node.op_type==OpType::SILU?2: + node.op_type==OpType::GELU_ERF?4: node.op_type==OpType::SIGMOID?5:3; if (in.precision == Precision::FP32 && out.precision == Precision::FP32) return cactus_metal_encode_unary_f32(code, out.get_data(), in.get_data(), out.total_size); if (!fp16(in) || !fp16(out)) return false; return cactus_metal_encode_unary(code, out.get_data(), in.get_data(), out.total_size); } + case OpType::SUM: case OpType::MEAN: case OpType::VARIANCE: + case OpType::MIN: case OpType::MAX: { + const auto& in = get_input(node, 0, nodes, map); + int axis = node.params.axis; + if (axis < 0 || (size_t)axis >= in.shape.size()) return false; + bool f32 = in.precision == Precision::FP32 && out.precision == Precision::FP32; + if (!f32 && (!fp16(in) || !fp16(out))) return false; + int code = node.op_type==OpType::SUM?0: node.op_type==OpType::MEAN?1: + node.op_type==OpType::VARIANCE?2: node.op_type==OpType::MIN?3:4; + size_t outer = 1, inner = 1; + for (size_t d = 0; d < (size_t)axis; ++d) outer *= in.shape[d]; + for (size_t d = (size_t)axis + 1; d < in.shape.size(); ++d) inner *= in.shape[d]; + return cactus_metal_encode_reduce_axis(code, out.get_data(), in.get_data(), + (uint32_t)outer, (uint32_t)in.shape[(size_t)axis], (uint32_t)inner, f32 ? 1 : 0); + } + case OpType::CUMSUM: { + const auto& in = get_input(node, 0, nodes, map); + int axis = node.params.axis; + if (axis < 0 || (size_t)axis >= in.shape.size()) return false; + bool f32 = in.precision == Precision::FP32 && out.precision == Precision::FP32; + if (!f32 && (!fp16(in) || !fp16(out))) return false; + size_t outer = 1, inner = 1; + for (size_t d = 0; d < (size_t)axis; ++d) outer *= in.shape[d]; + for (size_t d = (size_t)axis + 1; d < in.shape.size(); ++d) inner *= in.shape[d]; + return cactus_metal_encode_cumsum(out.get_data(), in.get_data(), + (uint32_t)outer, (uint32_t)in.shape[(size_t)axis], (uint32_t)inner, f32 ? 1 : 0); + } + case OpType::CONCAT: { + if (node.input_ids.size() != 2) return false; + const auto& a = get_input(node, 0, nodes, map); + const auto& b = get_input(node, 1, nodes, map); + if (!fp16(a) || !fp16(b) || !fp16(out)) return false; + int axis = node.params.axis; + if (axis < 0) axis += (int)a.shape.size(); + if (axis < 0 || (size_t)axis >= a.shape.size() || a.shape.size() != b.shape.size()) return false; + size_t outer = 1, inner = 1; + for (size_t d = 0; d < (size_t)axis; ++d) outer *= a.shape[d]; + for (size_t d = (size_t)axis + 1; d < a.shape.size(); ++d) inner *= a.shape[d]; + return cactus_metal_encode_concat2(out.get_data(), a.get_data(), b.get_data(), + (uint32_t)outer, (uint32_t)a.shape[(size_t)axis], (uint32_t)b.shape[(size_t)axis], (uint32_t)inner); + } + case OpType::GATHER: { + const auto& t = get_input(node, 0, nodes, map); + const auto& idx = get_input(node, 1, nodes, map); + if (!fp16(t) || !fp16(out) || idx.precision != Precision::FP32) return false; + if (t.shape.size() < 2) return false; + size_t D = t.total_size / t.shape[0]; + return cactus_metal_encode_gather_f32idx(out.get_data(), t.get_data(), idx.get_data(), + (uint32_t)idx.total_size, (uint32_t)D, t.byte_size); + } + case OpType::ROPE: case OpType::ROPE_GPTJ: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || !fp16(out) || in.shape.size() < 4) return false; + size_t S = in.shape[1], H = in.shape[2], D = in.shape[3]; + bool gptj = node.op_type == OpType::ROPE_GPTJ; + size_t rot = gptj ? (size_t)node.params.scalar : D; + if (rot == 0 || rot > D || (rot % 2) != 0) return false; + size_t tokens = in.total_size / D; + return cactus_metal_encode_rope_full(out.get_data(), in.get_data(), + (uint32_t)tokens, (uint32_t)S, (uint32_t)H, (uint32_t)D, (uint32_t)rot, + (uint32_t)node.params.position_offset, node.params.theta, gptj ? 1 : 0); + } + case OpType::MAXPOOL1D: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || !fp16(out) || in.shape.size() != 3 || out.shape.size() != 3) return false; + return cactus_metal_encode_maxpool1d(out.get_data(), in.get_data(), + (uint32_t)(in.shape[0] * in.shape[1]), (uint32_t)in.shape[2], + (uint32_t)out.shape[2], (uint32_t)node.params.kernel_size, (uint32_t)node.params.stride); + } + case OpType::BILINEAR_INTERPOLATION: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || !fp16(out) || in.shape.size() != 2) return false; + size_t src = (size_t)std::sqrt((double)in.shape[0]); + if (src * src != in.shape[0]) return false; + return cactus_metal_encode_bilinear(out.get_data(), in.get_data(), + (uint32_t)src, (uint32_t)src, (uint32_t)node.params.dst_height, + (uint32_t)node.params.dst_width, (uint32_t)in.shape[1], + node.params.align_corners ? 1 : 0); + } + case OpType::CONV1D: case OpType::CONV1D_K7S3: { + if (node.input_ids.size() < 2) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const BufferDesc* b = node.input_ids.size() > 2 ? &get_input(node, 2, nodes, map) : nullptr; + if (!fp16(x) || !fp16(w) || !fp16(out)) return false; + if (b && b->precision != Precision::FP16) return false; + if (x.shape.size() != 3 || w.shape.size() != 3 || out.shape.size() != 3) return false; + bool ck_co = node.op_type == OpType::CONV1D_K7S3; + size_t K = ck_co ? w.shape[1] : w.shape[2]; + size_t Cout = ck_co ? w.shape[2] : w.shape[0]; + return cactus_metal_encode_conv1d_gen(out.get_data(), x.get_data(), w.get_data(), + b ? b->get_data() : nullptr, (uint32_t)x.shape[0], (uint32_t)x.shape[1], + (uint32_t)x.shape[2], (uint32_t)Cout, (uint32_t)out.shape[2], (uint32_t)K, + (uint32_t)(node.params.stride ? node.params.stride : 1), ck_co ? 1 : 0); + } + case OpType::CONV1D_CAUSAL: case OpType::CONV1D_SAME_DEPTHWISE_K9: { + if (node.input_ids.size() < 2) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + bool causal = node.op_type == OpType::CONV1D_CAUSAL; + const BufferDesc* b = (!causal && node.input_ids.size() > 2) ? &get_input(node, 2, nodes, map) : nullptr; + if (!fp16(x) || !fp16(w) || !fp16(out)) return false; + if (b && b->precision != Precision::FP16) return false; + if (x.shape.size() != 3) return false; + size_t C = x.shape[2]; + size_t K; + if (causal) { + if (w.shape.size() != 3 || w.shape[0] != C || w.shape[1] != 1) return false; + K = w.shape[2]; + } else { + K = w.shape.back(); + if (w.total_size != C * K) return false; + } + size_t dil = causal ? (node.params.dilation ? node.params.dilation : 1) : 1; + size_t pad = causal ? (K - 1) * dil : K / 2; + return cactus_metal_encode_conv1d_nlc_dw(out.get_data(), x.get_data(), w.get_data(), + b ? b->get_data() : nullptr, (uint32_t)x.shape[0], (uint32_t)x.shape[1], + (uint32_t)C, (uint32_t)K, (uint32_t)dil, (uint32_t)pad); + } + case OpType::CONV2D_K3S2P1: case OpType::CONV2D_K3S1P1: + case OpType::CONV2D_DEPTHWISE_K3S2P1: case OpType::CONV2D_POINTWISE_1X1: { + if (node.input_ids.size() < 2) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const BufferDesc* b = node.input_ids.size() > 2 ? &get_input(node, 2, nodes, map) : nullptr; + if (!fp16(x) || !fp16(w) || !fp16(out)) return false; + if (b && b->precision != Precision::FP16) return false; + if (x.shape.size() != 4 || out.shape.size() != 4) return false; + bool dw = node.op_type == OpType::CONV2D_DEPTHWISE_K3S2P1; + bool pw = node.op_type == OpType::CONV2D_POINTWISE_1X1; + uint32_t K = pw ? 1 : 3; + uint32_t stride = (node.op_type == OpType::CONV2D_K3S1P1 || pw) ? 1 : 2; + uint32_t pad = pw ? 0 : 1; + return cactus_metal_encode_conv2d(out.get_data(), x.get_data(), w.get_data(), + b ? b->get_data() : nullptr, (uint32_t)x.shape[0], (uint32_t)x.shape[1], + (uint32_t)x.shape[2], (uint32_t)x.shape[3], (uint32_t)out.shape[1], + (uint32_t)out.shape[2], (uint32_t)out.shape[3], K, stride, pad, dw ? 1 : 0); + } + case OpType::CONV1D_POINTWISE: { + if (node.input_ids.size() < 2) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const BufferDesc* b = node.input_ids.size() > 2 ? &get_input(node, 2, nodes, map) : nullptr; + if (!fp16(x) || !fp16(w) || !fp16(out)) return false; + if (b && b->precision != Precision::FP16) return false; + if (x.shape.size() != 3 || out.shape.size() != 3) return false; + size_t Cin = x.shape[2], Cout = out.shape[2]; + if (w.total_size != Cin * Cout) return false; + size_t rows = x.shape[0] * x.shape[1]; + if (!cactus_metal_encode_gemm_f16(out.get_data(), x.get_data(), w.get_data(), + (uint32_t)rows, (uint32_t)Cin, (uint32_t)Cout, 1)) return false; + if (b) return cactus_metal_encode_bias_add_rows(out.get_data(), b->get_data(), + (uint32_t)Cout, (uint32_t)out.total_size); + return true; + } + case OpType::BATCHNORM: { + if (node.input_ids.size() != 5) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const auto& b = get_input(node, 2, nodes, map); + const auto& rm = get_input(node, 3, nodes, map); + const auto& rv = get_input(node, 4, nodes, map); + if (!fp16(x) || !fp16(out) || !fp16(w) || !fp16(b) || !fp16(rm) || !fp16(rv)) return false; + int axis = node.params.axis; + if (axis < 0) axis += (int)x.shape.size(); + if (axis < 0 || (size_t)axis >= x.shape.size()) return false; + size_t inner = 1; + for (size_t d = (size_t)axis + 1; d < x.shape.size(); ++d) inner *= x.shape[d]; + return cactus_metal_encode_batchnorm(out.get_data(), x.get_data(), w.get_data(), + b.get_data(), rm.get_data(), rv.get_data(), (uint32_t)x.shape[(size_t)axis], + (uint32_t)inner, (uint32_t)x.total_size, node.params.epsilon); + } + case OpType::GROUPNORM: { + if (node.input_ids.size() < 3) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const auto& b = get_input(node, 2, nodes, map); + if (!fp16(x) || !fp16(out) || !fp16(w) || !fp16(b) || x.shape.size() < 2) return false; + size_t spatial = 1; + for (size_t d = 2; d < x.shape.size(); ++d) spatial *= x.shape[d]; + size_t groups = node.params.num_groups ? node.params.num_groups : 32; + return cactus_metal_encode_groupnorm(out.get_data(), x.get_data(), w.get_data(), + b.get_data(), (uint32_t)x.shape[0], (uint32_t)x.shape[1], (uint32_t)spatial, + (uint32_t)groups, node.params.epsilon); + } case OpType::CLAMP: { const auto& in = get_input(node, 0, nodes, map); bool f32 = in.precision == Precision::FP32 && out.precision == Precision::FP32; @@ -908,6 +1113,7 @@ void CactusGraph::build_metal_retype_plan() { if (!ok[ri]) continue; const GraphNode& nd = *nodes_[ri]; if (retained_output_node_ids_.count(nd.id) || persistent_node_ids_.count(nd.id)) { ok[ri] = 0; continue; } + if (cons[ri].empty()) { ok[ri] = 0; continue; } for (size_t c : cons[ri]) { const GraphNode& cn = *nodes_[c]; bool absorbs = (cn.op_type == OpType::PRECISION_CAST diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 5681a3728..0eb602c4f 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -616,8 +616,15 @@ struct PaddedAppendBackup { uint64_t overshoot; uint64_t keep_sink; uint64_t kept_padded; + uint64_t ring_rows; }; +size_t ring_row_for_pos(size_t pos, size_t sink, size_t ring_capacity) { + if (pos < ring_capacity) return pos; + size_t recent = ring_capacity > sink ? ring_capacity - sink : 1; + return sink + (pos - sink) % recent; +} + struct CacheRowRegion { size_t offset; size_t row_bytes; @@ -649,6 +656,27 @@ std::vector CactusGraph::snapshot_cache_padded_append(size_t node_id, s const auto* meta = get_meta(buf); const size_t len0 = meta->current_seq_len; const size_t appended = real_tokens + pad_tokens; + if (kv_cache_resident()) { + const size_t ring_window = meta->max_seq_len - meta->sink_size - 1; + if (ring_window <= meta->sink_size || appended >= ring_window - meta->sink_size) { + throw std::runtime_error("padded cache append larger than the attention window is not supported"); + } + std::vector backup(sizeof(PaddedAppendBackup)); + auto* header = reinterpret_cast(backup.data()); + header->overshoot = 0; + header->keep_sink = meta->sink_size; + header->kept_padded = ring_window; + header->ring_rows = pad_tokens; + const auto* base = static_cast(buf.get_data()); + for (const auto& region : cache_row_regions(buf, meta)) { + for (size_t t = 0; t < pad_tokens; ++t) { + size_t row = ring_row_for_pos(len0 + real_tokens + t, meta->sink_size, ring_window); + const uint8_t* src = base + region.offset + row * region.row_bytes; + backup.insert(backup.end(), src, src + region.row_bytes); + } + } + return backup; + } if (len0 == 0 || len0 + appended <= window) return {}; const size_t keep_sink = std::min({static_cast(meta->sink_size), len0, window}); const size_t tail_capacity = window - keep_sink; @@ -686,6 +714,23 @@ void CactusGraph::rollback_cache_padded_append(size_t node_id, size_t real_token return; } const auto* header = reinterpret_cast(backup.data()); + if (header->ring_rows > 0) { + const size_t ring_window = header->kept_padded; + const size_t sink = header->keep_sink; + const size_t len_after = meta->current_seq_len; + const size_t len0 = len_after >= real_tokens + pad_tokens ? len_after - real_tokens - pad_tokens : 0; + auto* base = static_cast(buf.get_data()); + const uint8_t* saved = backup.data() + sizeof(PaddedAppendBackup); + for (const auto& region : cache_row_regions(buf, meta)) { + for (size_t t = 0; t < header->ring_rows; ++t) { + size_t row = ring_row_for_pos(len0 + real_tokens + t, sink, ring_window); + std::memcpy(base + region.offset + row * region.row_bytes, saved, region.row_bytes); + saved += region.row_bytes; + } + } + meta->current_seq_len = len0 + real_tokens; + return; + } const size_t overshoot = header->overshoot; const size_t keep_sink = header->keep_sink; const size_t kept_padded = header->kept_padded; diff --git a/cactus-graph/tests/test_metal_parity.cpp b/cactus-graph/tests/test_metal_parity.cpp new file mode 100644 index 000000000..d43f24434 --- /dev/null +++ b/cactus-graph/tests/test_metal_parity.cpp @@ -0,0 +1,317 @@ +#include "test_utils.h" +#include +#include +#include +#include +#include + +using namespace TestUtils; + +namespace { + +std::vector<__fp16> random_fp16(size_t n, float lo = -1.0f, float hi = 1.0f, uint32_t seed = 42) { + std::mt19937 gen(seed); + std::uniform_real_distribution dis(lo, hi); + std::vector<__fp16> out(n); + for (auto& v : out) v = static_cast<__fp16>(dis(gen)); + return out; +} + +struct ParityCase { + std::vector> input_shapes; + std::vector> input_data; + std::vector> input_data_f32; + std::function&)> build; + float tolerance = 5e-2f; + + size_t add_input(const std::vector& shape, float lo = -1.0f, float hi = 1.0f) { + size_t n = 1; + for (size_t d : shape) n *= d; + input_shapes.push_back(shape); + input_data.push_back(random_fp16(n, lo, hi, 42 + (uint32_t)input_shapes.size())); + input_data_f32.push_back({}); + return input_shapes.size() - 1; + } + + size_t add_input_f32(const std::vector& shape, const std::vector& data) { + input_shapes.push_back(shape); + input_data.push_back({}); + input_data_f32.push_back(data); + return input_shapes.size() - 1; + } + + std::vector run(const char* backend) { + if (cactus_backend_select(backend) != 0) return {}; + CactusGraph graph; + std::vector ids; + for (size_t i = 0; i < input_shapes.size(); ++i) { + bool f32 = !input_data_f32[i].empty(); + ids.push_back(graph.input(input_shapes[i], f32 ? Precision::FP32 : Precision::FP16)); + } + size_t out_id = build(graph, ids); + for (size_t i = 0; i < ids.size(); ++i) { + bool f32 = !input_data_f32[i].empty(); + graph.set_input(ids[i], + f32 ? static_cast(input_data_f32[i].data()) + : static_cast(input_data[i].data()), + f32 ? Precision::FP32 : Precision::FP16); + } + graph.execute(); + const auto& desc = graph.get_output_buffer(out_id); + std::vector result(desc.total_size); + if (desc.precision == Precision::FP16) { + const __fp16* p = static_cast(graph.get_output(out_id)); + for (size_t i = 0; i < result.size(); ++i) result[i] = static_cast(p[i]); + } else { + const float* p = static_cast(graph.get_output(out_id)); + for (size_t i = 0; i < result.size(); ++i) result[i] = p[i]; + } + graph.hard_reset(); + cactus_backend_select("auto"); + return result; + } + + bool check() { + std::vector cpu = run("cpu"); + std::vector metal = run("metal"); + if (metal.empty()) return true; + if (cpu.size() != metal.size() || cpu.empty()) return false; + for (size_t i = 0; i < cpu.size(); ++i) { + float scale = std::max(1.0f, std::fabs(cpu[i])); + if (std::fabs(cpu[i] - metal[i]) > tolerance * scale) { + std::cout << " mismatch at " << i << ": cpu=" << cpu[i] + << " metal=" << metal[i] << "\n"; + return false; + } + } + return true; + } +}; + +bool metal_present() { + bool ok = cactus_backend_select("metal") == 0; + cactus_backend_select("auto"); + return ok; +} + +bool parity_unary(size_t (CactusGraph::*fn)(size_t), float lo = -1.0f, float hi = 1.0f) { + ParityCase c; + c.add_input({4, 33}, lo, hi); + c.build = [fn](CactusGraph& g, const std::vector& in) { return (g.*fn)(in[0]); }; + return c.check(); +} + +bool parity_scalar(size_t (CactusGraph::*fn)(size_t, float), float p, float lo = -1.0f, float hi = 1.0f) { + ParityCase c; + c.add_input({4, 33}, lo, hi); + c.build = [fn, p](CactusGraph& g, const std::vector& in) { return (g.*fn)(in[0], p); }; + return c.check(); +} + +bool parity_reduce(size_t (CactusGraph::*fn)(size_t, int)) { + for (int axis = 0; axis < 3; ++axis) { + ParityCase c; + c.add_input({3, 5, 17}); + c.build = [fn, axis](CactusGraph& g, const std::vector& in) { return (g.*fn)(in[0], axis); }; + if (!c.check()) return false; + } + return true; +} + +bool parity_not_equal() { + ParityCase c; + c.add_input({2, 31}); + c.add_input({2, 31}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.not_equal(in[0], in[1]); }; + return c.check(); +} + +bool parity_concat() { + for (int axis = 0; axis < 2; ++axis) { + ParityCase c; + c.add_input({3, 7}); + c.add_input({axis == 0 ? 5u : 3u, axis == 0 ? 7u : 11u}); + c.build = [axis](CactusGraph& g, const std::vector& in) { return g.concat(in[0], in[1], axis); }; + if (!c.check()) return false; + } + return true; +} + +bool parity_gather() { + ParityCase c; + c.add_input({16, 24}); + c.add_input_f32({6}, {3.0f, 0.0f, 15.0f, 7.0f, 7.0f, 2.0f}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.gather(in[0], in[1]); }; + return c.check(); +} + +bool parity_rope(bool gptj) { + ParityCase c; + c.add_input({1, 6, 4, 32}); + c.build = [gptj](CactusGraph& g, const std::vector& in) { + return gptj ? g.rope_gptj(in[0], 10000.0f, 3, 16) : g.rope(in[0], 10000.0f, 3); + }; + return c.check(); +} + +bool parity_maxpool1d() { + ParityCase c; + c.add_input({2, 5, 29}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.maxpool1d(in[0], 3, 2); }; + return c.check(); +} + +bool parity_bilinear() { + ParityCase c; + c.add_input({64, 12}); + c.build = [](CactusGraph& g, const std::vector& in) { + return g.bilinear_interpolation(in[0], 13, 11, false); + }; + return c.check(); +} + +bool parity_conv1d() { + ParityCase c; + c.add_input({2, 6, 31}); + c.add_input({4, 6, 5}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.conv1d(in[0], in[1], 2); }; + return c.check(); +} + +bool parity_conv1d_k7s3() { + ParityCase c; + c.add_input({1, 8, 40}); + c.add_input({8, 7, 16}); + c.add_input({16}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.conv1d_k7s3(in[0], in[1], in[2]); }; + return c.check(); +} + +bool parity_conv1d_causal() { + for (size_t dil : {1u, 2u}) { + ParityCase c; + c.add_input({1, 21, 8}); + c.add_input({8, 1, 4}); + c.build = [dil](CactusGraph& g, const std::vector& in) { + return g.conv1d_causal(in[0], in[1], 4, dil); + }; + if (!c.check()) return false; + } + return true; +} + +bool parity_conv1d_dw_k9() { + ParityCase c; + c.add_input({1, 25, 6}); + c.add_input({6, 1, 9}); + c.add_input({6}); + c.build = [](CactusGraph& g, const std::vector& in) { + return g.conv1d_same_depthwise_k9(in[0], in[1], in[2]); + }; + return c.check(); +} + +bool parity_conv1d_pointwise() { + ParityCase c; + c.add_input({1, 13, 16}); + c.add_input({24, 16}); + c.add_input({24}); + c.build = [](CactusGraph& g, const std::vector& in) { + return g.conv1d_pointwise(in[0], in[1], in[2]); + }; + return c.check(); +} + +bool parity_conv2d(size_t (CactusGraph::*fn)(size_t, size_t, size_t), bool depthwise, bool pointwise) { + ParityCase c; + c.add_input({1, 4, 11, 13}); + if (depthwise) c.add_input({4, 1, 3, 3}); + else if (pointwise) c.add_input({6, 4, 1, 1}); + else c.add_input({6, 4, 3, 3}); + c.add_input({pointwise || depthwise ? (depthwise ? 4u : 6u) : 6u}); + c.build = [fn](CactusGraph& g, const std::vector& in) { return (g.*fn)(in[0], in[1], in[2]); }; + return c.check(); +} + +bool parity_batchnorm() { + ParityCase c; + c.add_input({2, 5, 9}); + c.add_input({5}); + c.add_input({5}); + c.add_input({5}); + c.add_input({5}, 0.5f, 1.5f); + c.build = [](CactusGraph& g, const std::vector& in) { + return g.batchnorm(in[0], in[1], in[2], in[3], in[4], 1); + }; + return c.check(); +} + +bool parity_groupnorm() { + ParityCase c; + c.add_input({2, 8, 6}); + c.add_input({8}); + c.add_input({8}); + c.build = [](CactusGraph& g, const std::vector& in) { + return g.groupnorm(in[0], in[1], in[2], 4); + }; + return c.check(); +} + +bool parity_cumsum() { + for (int axis = 0; axis < 2; ++axis) { + ParityCase c; + c.add_input({4, 19}); + c.build = [axis](CactusGraph& g, const std::vector& in) { return g.cumsum(in[0], axis); }; + if (!c.check()) return false; + } + return true; +} + +} + +int main() { + TestRunner runner("Metal Parity (CPU vs Metal per-op)"); + if (!metal_present()) { + std::cout << "Metal backend unavailable; skipping parity suite.\n"; + return 0; + } + + runner.run_test("abs", parity_unary(&CactusGraph::abs)); + runner.run_test("scalar_exp", parity_unary(&CactusGraph::scalar_exp)); + runner.run_test("scalar_sqrt", parity_unary(&CactusGraph::scalar_sqrt, 0.1f, 2.0f)); + runner.run_test("scalar_cos", parity_unary(&CactusGraph::scalar_cos)); + runner.run_test("scalar_sin", parity_unary(&CactusGraph::scalar_sin)); + runner.run_test("scalar_log", parity_unary(&CactusGraph::scalar_log, 0.1f, 3.0f)); + runner.run_test("gelu_erf", parity_unary(&CactusGraph::gelu_erf)); + runner.run_test("sigmoid", parity_unary(&CactusGraph::sigmoid)); + runner.run_test("pow", parity_scalar(&CactusGraph::pow, 2.0f, 0.1f, 2.0f)); + runner.run_test("leaky_relu", parity_scalar(&CactusGraph::leaky_relu, 0.1f)); + runner.run_test("scalar_not_equal", parity_scalar(&CactusGraph::scalar_not_equal, 0.0f)); + runner.run_test("not_equal", parity_not_equal()); + runner.run_test("sum", parity_reduce(&CactusGraph::sum)); + runner.run_test("mean", parity_reduce(&CactusGraph::mean)); + runner.run_test("variance", parity_reduce(&CactusGraph::variance)); + runner.run_test("min", parity_reduce(&CactusGraph::min)); + runner.run_test("max", parity_reduce(&CactusGraph::max)); + runner.run_test("cumsum", parity_cumsum()); + runner.run_test("concat", parity_concat()); + runner.run_test("gather", parity_gather()); + runner.run_test("rope", parity_rope(false)); + runner.run_test("rope_gptj", parity_rope(true)); + runner.run_test("maxpool1d", parity_maxpool1d()); + runner.run_test("bilinear_interpolation", parity_bilinear()); + runner.run_test("conv1d", parity_conv1d()); + runner.run_test("conv1d_k7s3", parity_conv1d_k7s3()); + runner.run_test("conv1d_causal", parity_conv1d_causal()); + runner.run_test("conv1d_same_depthwise_k9", parity_conv1d_dw_k9()); + runner.run_test("conv1d_pointwise", parity_conv1d_pointwise()); + runner.run_test("conv2d_k3s2p1", parity_conv2d(&CactusGraph::conv2d_k3s2p1, false, false)); + runner.run_test("conv2d_k3s1p1", parity_conv2d(&CactusGraph::conv2d_k3s1p1, false, false)); + runner.run_test("conv2d_depthwise_k3s2p1", parity_conv2d(&CactusGraph::conv2d_depthwise_k3s2p1, true, false)); + runner.run_test("conv2d_pointwise_1x1", parity_conv2d(&CactusGraph::conv2d_pointwise_1x1, false, true)); + runner.run_test("batchnorm", parity_batchnorm()); + runner.run_test("groupnorm", parity_groupnorm()); + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index a29c60076..627ce932e 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -117,6 +117,37 @@ bool cactus_metal_encode_gemm_f16(void* out, const void* lhs, const void* rhs, bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, const void* v, const void* mask, uint32_t B, uint32_t T, uint32_t S, uint32_t HQ, uint32_t HKV, uint32_t D, uint32_t DV, float scale, uint32_t causal, uint32_t pos_off, uint32_t window, float logit_cap, uint32_t mask_mode); +bool cactus_metal_encode_reduce_axis(int op, void* out, const void* in, uint32_t outer, + uint32_t axis_size, uint32_t inner, int f32); +bool cactus_metal_encode_cumsum(void* out, const void* in, uint32_t outer, + uint32_t axis_size, uint32_t inner, int f32); +bool cactus_metal_encode_concat2(void* out, const void* a, const void* b, + uint32_t outer, uint32_t a_axis, uint32_t b_axis, uint32_t inner); +bool cactus_metal_encode_gather_f32idx(void* out, const void* table, const void* idx, + uint32_t rows, uint32_t D, size_t table_bytes); +bool cactus_metal_encode_rope_full(void* out, const void* in, uint32_t tokens, uint32_t S, + uint32_t H, uint32_t D, uint32_t rot, uint32_t pos0, + float theta, int gptj); +bool cactus_metal_encode_maxpool1d(void* out, const void* in, uint32_t NC, uint32_t L, + uint32_t Lout, uint32_t K, uint32_t stride); +bool cactus_metal_encode_bilinear(void* out, const void* in, uint32_t sh, uint32_t sw, + uint32_t dh, uint32_t dw, uint32_t E, int align); +bool cactus_metal_encode_conv1d_gen(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t Cin, uint32_t L, uint32_t Cout, + uint32_t Lout, uint32_t K, uint32_t stride, int w_ck_co); +bool cactus_metal_encode_conv1d_nlc_dw(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t L, uint32_t C, uint32_t K, + uint32_t dil, uint32_t pad); +bool cactus_metal_encode_conv2d(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t Cin, uint32_t H, uint32_t W, uint32_t Cout, + uint32_t Ho, uint32_t Wo, uint32_t K, uint32_t stride, + uint32_t pad, int dw); +bool cactus_metal_encode_batchnorm(void* out, const void* x, const void* w, const void* b, + const void* rm, const void* rv, uint32_t C, uint32_t inner, + uint32_t total, float eps); +bool cactus_metal_encode_groupnorm(void* out, const void* x, const void* w, const void* b, + uint32_t N, uint32_t C, uint32_t S, uint32_t groups, float eps); +bool cactus_metal_encode_bias_add_rows(void* y, const void* bias, uint32_t C, uint32_t total); bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a); bool cactus_metal_encode_conv1d_dw(void* out, const void* x, const void* w, diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index cbee12de0..a627f7b91 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -930,7 +930,8 @@ kernel void binary_f16(device const half* a [[buffer(0)]], device const half* b device half* y [[buffer(2)]], constant uint& n [[buffer(3)]], constant int& op [[buffer(4)]], uint i [[thread_position_in_grid]]) { if (i>=n) return; float av=(float)a[i], bv=(float)b[i], r; - switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; default: r=av+bv; } + switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; + case 5: r=(av!=bv)?1.0f:0.0f; break; default: r=av+bv; } if (op==1) r=clamp(r,-65500.0f,65500.0f); y[i]=(half)r; } @@ -976,12 +977,38 @@ kernel void rms_norm_rope_f16( } } +static inline float erf_approx(float x) { + float sgn = x < 0.0f ? -1.0f : 1.0f; + float ax = fabs(x); + float t = 1.0f / (1.0f + 0.3275911f * ax); + float poly = t * (0.254829592f + t * (-0.284496736f + t * (1.421413741f + + t * (-1.453152027f + t * 1.061405429f)))); + return sgn * (1.0f - poly * precise::exp(-ax * ax)); +} + +static inline float scalar_apply(float v, float p, int op) { + switch (op) { + case 0: return v+p; + case 1: return v-p; + case 3: return v/p; + case 4: return precise::exp(v); + case 5: return precise::sqrt(v); + case 6: return precise::cos(v); + case 7: return precise::sin(v); + case 8: return precise::log(v); + case 9: return fabs(v); + case 10: return precise::pow(v, p); + case 11: return (v != p) ? 1.0f : 0.0f; + case 12: return (v > 0.0f) ? v : v*p; + default: return v*p; + } +} + kernel void scalar_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], constant float& p [[buffer(4)]], uint i [[thread_position_in_grid]]) { - if (i>=n) return; float v=(float)in[i], r; - switch(op){ case 0: r=v+p; break; case 1: r=v-p; break; case 3: r=v/p; break; default: r=v*p; } - y[i]=(half)r; + if (i>=n) return; + y[i]=(half)scalar_apply((float)in[i], p, op); } kernel void unary_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], @@ -991,6 +1018,8 @@ kernel void unary_f16(device const half* in [[buffer(0)]], device half* y [[buff if (op==0) r=gelu_tanh(x); else if (op==1) r=precise::tanh(x); else if (op==2) r=x/(1.0f+precise::exp(-x)); + else if (op==4) r=0.5f*x*(1.0f+erf_approx(x*0.70710678f)); + else if (op==5) r=1.0f/(1.0f+precise::exp(-x)); else r=max(x,0.0f); y[i]=(half)r; } @@ -1203,7 +1232,7 @@ kernel void binary_f32(device const float* a [[buffer(0)]], device const float* device float* y [[buffer(2)]], constant uint& n [[buffer(3)]], constant int& op [[buffer(4)]], uint i [[thread_position_in_grid]]) { if (i>=n) return; float av=a[i], bv=b[i], r; - switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; default: r=av+bv; } + switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; case 5: r=(av!=bv)?1.0f:0.0f; break; default: r=av+bv; } if (op==1) r=clamp(r,-65500.0f,65500.0f); y[i]=r; } @@ -1211,9 +1240,8 @@ kernel void binary_f32(device const float* a [[buffer(0)]], device const float* kernel void scalar_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], constant float& p [[buffer(4)]], uint i [[thread_position_in_grid]]) { - if (i>=n) return; float v=in[i], r; - switch(op){ case 0: r=v+p; break; case 1: r=v-p; break; case 3: r=v/p; break; default: r=v*p; } - y[i]=r; + if (i>=n) return; + y[i]=scalar_apply(in[i], p, op); } kernel void unary_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], @@ -1223,6 +1251,8 @@ kernel void unary_f32(device const float* in [[buffer(0)]], device float* y [[bu if (op==0) r=gelu_tanh(x); else if (op==1) r=precise::tanh(x); else if (op==2) r=x/(1.0f+precise::exp(-x)); + else if (op==4) r=0.5f*x*(1.0f+erf_approx(x*0.70710678f)); + else if (op==5) r=1.0f/(1.0f+precise::exp(-x)); else r=max(x,0.0f); y[i]=r; } @@ -1579,6 +1609,288 @@ kernel void conv1d_dw_f16(device const half* x [[buffer(0)]], device const half* y[(size_t)c*Lout + l] = (half)acc; } +kernel void reduce_axis_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& axis_size [[buffer(2)]], constant uint& inner [[buffer(3)]], + constant uint& n [[buffer(4)]], constant int& op [[buffer(5)]], + uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint outer = i / inner, in_i = i % inner; + size_t base = (size_t)outer * axis_size * inner + in_i; + float r; + if (op <= 2) { + float sum = 0; + for (uint a = 0; a < axis_size; ++a) sum += (float)in[base + (size_t)a*inner]; + float mean = sum / (float)max(axis_size, 1u); + if (op == 0) r = sum; + else if (op == 1) r = mean; + else { + float var = 0; + for (uint a = 0; a < axis_size; ++a) { float d = (float)in[base + (size_t)a*inner] - mean; var += d*d; } + r = var / (float)max(axis_size, 1u); + } + } else { + r = axis_size ? (float)in[base] : 0.0f; + for (uint a = 1; a < axis_size; ++a) { + float v = (float)in[base + (size_t)a*inner]; + r = (op == 3) ? min(r, v) : max(r, v); + } + } + y[i] = (half)r; +} + +kernel void reduce_axis_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], + constant uint& axis_size [[buffer(2)]], constant uint& inner [[buffer(3)]], + constant uint& n [[buffer(4)]], constant int& op [[buffer(5)]], + uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint outer = i / inner, in_i = i % inner; + size_t base = (size_t)outer * axis_size * inner + in_i; + float r; + if (op <= 2) { + float sum = 0; + for (uint a = 0; a < axis_size; ++a) sum += in[base + (size_t)a*inner]; + float mean = sum / (float)max(axis_size, 1u); + if (op == 0) r = sum; + else if (op == 1) r = mean; + else { + float var = 0; + for (uint a = 0; a < axis_size; ++a) { float d = in[base + (size_t)a*inner] - mean; var += d*d; } + r = var / (float)max(axis_size, 1u); + } + } else { + r = axis_size ? in[base] : 0.0f; + for (uint a = 1; a < axis_size; ++a) { + float v = in[base + (size_t)a*inner]; + r = (op == 3) ? min(r, v) : max(r, v); + } + } + y[i] = r; +} + +kernel void cumsum_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& axis_size [[buffer(2)]], constant uint& inner [[buffer(3)]], + constant uint& n [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint outer = i / inner, in_i = i % inner; + size_t base = (size_t)outer * axis_size * inner + in_i; + float run = 0; + for (uint a = 0; a < axis_size; ++a) { + run += (float)in[base + (size_t)a*inner]; + y[base + (size_t)a*inner] = (half)run; + } +} + +kernel void cumsum_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], + constant uint& axis_size [[buffer(2)]], constant uint& inner [[buffer(3)]], + constant uint& n [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint outer = i / inner, in_i = i % inner; + size_t base = (size_t)outer * axis_size * inner + in_i; + float run = 0; + for (uint a = 0; a < axis_size; ++a) { + run += in[base + (size_t)a*inner]; + y[base + (size_t)a*inner] = run; + } +} + +kernel void concat2_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], + device half* y [[buffer(2)]], constant uint& a_axis [[buffer(3)]], + constant uint& b_axis [[buffer(4)]], constant uint& inner [[buffer(5)]], + constant uint& n [[buffer(6)]], uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint in_i = i % inner, rest = i / inner; + uint ab = a_axis + b_axis; + uint ax = rest % ab, outer = rest / ab; + y[i] = (ax < a_axis) ? a[((size_t)outer*a_axis + ax)*inner + in_i] + : b[((size_t)outer*b_axis + (ax - a_axis))*inner + in_i]; +} + +kernel void gather_f32idx_f16(device const half* table [[buffer(0)]], device const float* rows [[buffer(1)]], + device half* out [[buffer(2)]], constant uint& D [[buffer(3)]], + constant uint& n [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint row = i / D, d = i % D; + out[i] = table[(size_t)(uint)rows[row]*D + d]; +} + +kernel void rope_full_f16(device const half* x [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& S [[buffer(2)]], constant uint& H [[buffer(3)]], + constant uint& D [[buffer(4)]], constant uint& rot [[buffer(5)]], + constant uint& pos0 [[buffer(6)]], constant float& theta [[buffer(7)]], + constant uint& gptj [[buffer(8)]], + uint2 gid [[thread_position_in_grid]]) { + uint d = gid.x, tok = gid.y; + if (d >= D) return; + size_t off = (size_t)tok * D + d; + uint hd = D; + uint seq = (tok / H) % S; + if (d >= rot) { y[off] = x[off]; return; } + uint hr = rot / 2; + uint pair_i = gptj ? d / 2 : (d % hr); + float freq = 1.0f / precise::pow(theta, (2.0f * pair_i) / (float)rot); + float angle = (float)(pos0 + seq) * freq; + float c = (float)(half)precise::cos(angle); + float sn = (float)(half)precise::sin(angle); + float v = (float)x[off]; + float other; + bool first; + if (gptj) { first = (d % 2) == 0; other = (float)x[off + (first ? 1 : -1)]; } + else { first = d < hr; other = (float)x[off + (first ? (long)hr : -(long)hr)]; } + y[off] = (half)(first ? v*c - other*sn : v*c + other*sn); + (void)hd; +} + +kernel void maxpool1d_f16(device const half* x [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& L [[buffer(2)]], constant uint& Lout [[buffer(3)]], + constant uint& K [[buffer(4)]], constant uint& stride [[buffer(5)]], + uint2 gid [[thread_position_in_grid]]) { + uint lo = gid.x, nc = gid.y; + if (lo >= Lout) return; + size_t base = (size_t)nc * L + (size_t)lo * stride; + float r = -INFINITY; + for (uint k = 0; k < K && lo * stride + k < L; ++k) r = max(r, (float)x[base + k]); + y[(size_t)nc * Lout + lo] = (half)r; +} + +kernel void bilinear_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& sh [[buffer(2)]], constant uint& sw [[buffer(3)]], + constant uint& dh [[buffer(4)]], constant uint& dw [[buffer(5)]], + constant uint& E [[buffer(6)]], constant uint& align [[buffer(7)]], + uint2 gid [[thread_position_in_grid]]) { + uint e = gid.x, pix = gid.y; + if (e >= E || pix >= dh * dw) return; + uint dy = pix / dw, dx = pix % dw; + float syf, sxf; + if (align) { + float sc_h = (sh > 1 && dh > 1) ? (float)(sh - 1) / (float)(dh - 1) : 0.0f; + float sc_w = (sw > 1 && dw > 1) ? (float)(sw - 1) / (float)(dw - 1) : 0.0f; + syf = dy * sc_h; sxf = dx * sc_w; + } else { + float sc_h = dh ? (float)sh / (float)dh : 0.0f; + float sc_w = dw ? (float)sw / (float)dw : 0.0f; + syf = clamp((dy + 0.5f) * sc_h - 0.5f, 0.0f, (float)sh - 1.0f); + sxf = clamp((dx + 0.5f) * sc_w - 0.5f, 0.0f, (float)sw - 1.0f); + } + int y0 = (int)floor(syf), x0 = (int)floor(sxf); + int y1 = min(y0 + 1, (int)sh - 1), x1 = min(x0 + 1, (int)sw - 1); + float fy = syf - y0, fx = sxf - x0; + float v00 = (float)in[((size_t)y0*sw + x0)*E + e], v01 = (float)in[((size_t)y0*sw + x1)*E + e]; + float v10 = (float)in[((size_t)y1*sw + x0)*E + e], v11 = (float)in[((size_t)y1*sw + x1)*E + e]; + y[(size_t)pix*E + e] = (half)(v00*(1-fx)*(1-fy) + v01*fx*(1-fy) + v10*(1-fx)*fy + v11*fx*fy); +} + +kernel void conv1d_gen_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* bias [[buffer(2)]], device half* y [[buffer(3)]], + constant uint& Cin [[buffer(4)]], constant uint& L [[buffer(5)]], + constant uint& Cout [[buffer(6)]], constant uint& Lout [[buffer(7)]], + constant uint& K [[buffer(8)]], constant uint& stride [[buffer(9)]], + constant uint& has_bias [[buffer(10)]], constant uint& w_ck_co [[buffer(11)]], + uint3 gid [[thread_position_in_grid]]) { + uint lo = gid.x, co = gid.y, n = gid.z; + if (lo >= Lout || co >= Cout) return; + float acc = has_bias ? (float)bias[co] : 0.0f; + for (uint ci = 0; ci < Cin; ++ci) { + device const half* xr = x + ((size_t)n*Cin + ci)*L + (size_t)lo*stride; + for (uint k = 0; k < K; ++k) { + float wv = w_ck_co ? (float)w[((size_t)ci*K + k)*Cout + co] + : (float)w[((size_t)co*Cin + ci)*K + k]; + acc = fma((float)xr[k], wv, acc); + } + } + y[((size_t)n*Cout + co)*Lout + lo] = (half)acc; +} + +kernel void conv1d_nlc_dw_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* bias [[buffer(2)]], device half* y [[buffer(3)]], + constant uint& L [[buffer(4)]], constant uint& C [[buffer(5)]], + constant uint& K [[buffer(6)]], constant uint& dil [[buffer(7)]], + constant uint& pad [[buffer(8)]], constant uint& has_bias [[buffer(9)]], + uint3 gid [[thread_position_in_grid]]) { + uint c = gid.x, l = gid.y, n = gid.z; + if (c >= C || l >= L) return; + float acc = has_bias ? (float)bias[c] : 0.0f; + for (uint k = 0; k < K; ++k) { + long p = (long)l - (long)pad + (long)k * dil; + if (p >= 0 && p < (long)L) + acc = fma((float)x[((size_t)n*L + p)*C + c], (float)w[(size_t)c*K + k], acc); + } + y[((size_t)n*L + l)*C + c] = (half)acc; +} + +kernel void conv2d_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* bias [[buffer(2)]], device half* y [[buffer(3)]], + constant uint& Cin [[buffer(4)]], constant uint& H [[buffer(5)]], + constant uint& W [[buffer(6)]], constant uint& Cout [[buffer(7)]], + constant uint& Ho [[buffer(8)]], constant uint& Wo [[buffer(9)]], + constant uint& K [[buffer(10)]], constant uint& stride [[buffer(11)]], + constant uint& pad [[buffer(12)]], constant uint& dw [[buffer(13)]], + constant uint& has_bias [[buffer(14)]], + uint3 gid [[thread_position_in_grid]]) { + uint wo = gid.x, ho = gid.y, nco = gid.z; + if (wo >= Wo || ho >= Ho) return; + uint n = nco / Cout, co = nco % Cout; + float acc = has_bias ? (float)bias[co] : 0.0f; + uint ci0 = dw ? co : 0, ci1 = dw ? co + 1 : Cin; + for (uint ci = ci0; ci < ci1; ++ci) { + for (uint kh = 0; kh < K; ++kh) { + long h = (long)ho * stride - (long)pad + kh; + if (h < 0 || h >= (long)H) continue; + for (uint kw = 0; kw < K; ++kw) { + long ww = (long)wo * stride - (long)pad + kw; + if (ww < 0 || ww >= (long)W) continue; + float wv = dw ? (float)w[((size_t)co*K + kh)*K + kw] + : (float)w[(((size_t)co*Cin + ci)*K + kh)*K + kw]; + acc = fma((float)x[(((size_t)n*Cin + ci)*H + h)*W + ww], wv, acc); + } + } + } + y[(((size_t)n*Cout + co)*Ho + ho)*Wo + wo] = (half)acc; +} + +kernel void batchnorm_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* b [[buffer(2)]], device const half* rm [[buffer(3)]], + device const half* rv [[buffer(4)]], device half* y [[buffer(5)]], + constant uint& C [[buffer(6)]], constant uint& inner [[buffer(7)]], + constant float& eps [[buffer(8)]], constant uint& n [[buffer(9)]], + uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint c = (i / inner) % C; + float inv = 1.0f / precise::sqrt((float)rv[c] + eps); + y[i] = (half)(((float)x[i] - (float)rm[c]) * inv * (float)w[c] + (float)b[c]); +} + +kernel void groupnorm_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* b [[buffer(2)]], device half* y [[buffer(3)]], + constant uint& cpg [[buffer(4)]], constant uint& S [[buffer(5)]], + constant uint& C [[buffer(6)]], constant float& eps [[buffer(7)]], + uint2 tgp [[threadgroup_position_in_grid]], + uint2 tp [[thread_position_in_threadgroup]], uint2 ntp [[threads_per_threadgroup]], + threadgroup float* red [[threadgroup(0)]]) { + uint g = tgp.x, n = tgp.y, t = tp.x, nt = ntp.x; + size_t base = ((size_t)n*C + (size_t)g*cpg) * S; + uint count = cpg * S; + float sum = 0, sq = 0; + for (uint i = t; i < count; i += nt) { float v = (float)x[base + i]; sum += v; sq += v*v; } + red[t] = sum; threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint s2 = nt/2; s2 > 0; s2 >>= 1) { if (t < s2) red[t] += red[t+s2]; threadgroup_barrier(mem_flags::mem_threadgroup); } + sum = red[0]; threadgroup_barrier(mem_flags::mem_threadgroup); + red[t] = sq; threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint s2 = nt/2; s2 > 0; s2 >>= 1) { if (t < s2) red[t] += red[t+s2]; threadgroup_barrier(mem_flags::mem_threadgroup); } + sq = red[0]; + float mean = sum / count; + float inv = 1.0f / precise::sqrt(sq / count - mean*mean + eps); + for (uint i = t; i < count; i += nt) { + uint c = g*cpg + i / S; + y[base + i] = (half)(((float)x[base + i] - mean) * inv * (float)w[c] + (float)b[c]); + } +} + +kernel void bias_add_rows_f16(device half* y [[buffer(0)]], device const half* bias [[buffer(1)]], + constant uint& C [[buffer(2)]], constant uint& n [[buffer(3)]], + uint i [[thread_position_in_grid]]) { + if (i < n) y[i] = (half)((float)y[i] + (float)bias[i % C]); +} + kernel void bcast_binary_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], device half* out [[buffer(2)]], constant uint* oshape [[buffer(3)]], constant uint* astride [[buffer(4)]], constant uint* bstride [[buffer(5)]], diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index e636cdff2..6444eb5e5 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -51,6 +51,10 @@ id psoClampF16=nil, psoClampF32=nil, psoGlu=nil; id psoLayerNorm=nil, psoSoftmaxR=nil, psoConv1dK3=nil, psoAttnDense=nil, psoAttnD64=nil; id psoGemmBatch=nil, psoGemmBatchF32A=nil, psoConvDw=nil, psoStridedRows=nil, psoMaskFill=nil; + id psoReduceF16=nil, psoReduceF32=nil, psoCumsumF16=nil, psoCumsumF32=nil; + id psoConcat2=nil, psoGatherF32Idx=nil, psoRopeFull=nil, psoMaxpool1d=nil; + id psoBilinear=nil, psoConv1dGen=nil, psoConv1dNlcDw=nil, psoConv2d=nil; + id psoBatchnorm=nil, psoGroupnorm=nil, psoBiasAddRows=nil; id dummy=nil; bool ok = false; @@ -98,6 +102,14 @@ psoConv1dK3=pso("conv1d_k3_f16"); psoAttnDense=pso("attn_f16"); psoAttnD64=pso("attn_f16_d64"); psoGemmBatch=pso("gemm_batch_f16"); psoGemmBatchF32A=pso("gemm_batch_f32a"); psoConvDw=pso("conv1d_dw_f16"); psoStridedRows=pso("strided_copy_rows_f16"); psoMaskFill=pso("attn_maskfill_f16"); + psoReduceF16=pso("reduce_axis_f16"); psoReduceF32=pso("reduce_axis_f32"); + psoCumsumF16=pso("cumsum_f16"); psoCumsumF32=pso("cumsum_f32"); + psoConcat2=pso("concat2_f16"); psoGatherF32Idx=pso("gather_f32idx_f16"); + psoRopeFull=pso("rope_full_f16"); psoMaxpool1d=pso("maxpool1d_f16"); + psoBilinear=pso("bilinear_f16"); psoConv1dGen=pso("conv1d_gen_f16"); + psoConv1dNlcDw=pso("conv1d_nlc_dw_f16"); psoConv2d=pso("conv2d_f16"); + psoBatchnorm=pso("batchnorm_f16"); psoGroupnorm=pso("groupnorm_f16"); + psoBiasAddRows=pso("bias_add_rows_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; @@ -1507,6 +1519,186 @@ bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, return true; } +bool cactus_metal_encode_reduce_axis(int op, void* out, const void* in, uint32_t outer, + uint32_t axis_size, uint32_t inner, int f32) { + if (!ctx().ok || !ctx().psoReduceF16 || !ctx().psoReduceF32) return false; + ensureEncoder(); + size_t es = f32 ? 4 : 2; + uint32_t n = outer * inner; + [g_enc setComputePipelineState:(f32 ? ctx().psoReduceF32 : ctx().psoReduceF16)]; + setBufAt(in, (size_t)outer*axis_size*inner*es, 0); setBufAt(out, (size_t)n*es, 1); + [g_enc setBytes:&axis_size length:4 atIndex:2]; [g_enc setBytes:&inner length:4 atIndex:3]; + [g_enc setBytes:&n length:4 atIndex:4]; [g_enc setBytes:&op length:4 atIndex:5]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_cumsum(void* out, const void* in, uint32_t outer, + uint32_t axis_size, uint32_t inner, int f32) { + if (!ctx().ok || !ctx().psoCumsumF16 || !ctx().psoCumsumF32) return false; + ensureEncoder(); + size_t es = f32 ? 4 : 2; + uint32_t n = outer * inner; + size_t bytes = (size_t)outer*axis_size*inner*es; + [g_enc setComputePipelineState:(f32 ? ctx().psoCumsumF32 : ctx().psoCumsumF16)]; + setBufAt(in, bytes, 0); setBufAt(out, bytes, 1); + [g_enc setBytes:&axis_size length:4 atIndex:2]; [g_enc setBytes:&inner length:4 atIndex:3]; + [g_enc setBytes:&n length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_concat2(void* out, const void* a, const void* b, + uint32_t outer, uint32_t a_axis, uint32_t b_axis, uint32_t inner) { + if (!ctx().ok || !ctx().psoConcat2) return false; + ensureEncoder(); + uint32_t n = outer * (a_axis + b_axis) * inner; + [g_enc setComputePipelineState:ctx().psoConcat2]; + setBufAt(a, (size_t)outer*a_axis*inner*2, 0); setBufAt(b, (size_t)outer*b_axis*inner*2, 1); + setBufAt(out, (size_t)n*2, 2); + [g_enc setBytes:&a_axis length:4 atIndex:3]; [g_enc setBytes:&b_axis length:4 atIndex:4]; + [g_enc setBytes:&inner length:4 atIndex:5]; [g_enc setBytes:&n length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_gather_f32idx(void* out, const void* table, const void* idx, + uint32_t rows, uint32_t D, size_t table_bytes) { + if (!ctx().ok || !ctx().psoGatherF32Idx) return false; + ensureEncoder(); + uint32_t n = rows * D; + [g_enc setComputePipelineState:ctx().psoGatherF32Idx]; + setBufAt(table, table_bytes, 0); setBufAt(idx, (size_t)rows*4, 1); setBufAt(out, (size_t)n*2, 2); + [g_enc setBytes:&D length:4 atIndex:3]; [g_enc setBytes:&n length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_rope_full(void* out, const void* in, uint32_t tokens, uint32_t S, + uint32_t H, uint32_t D, uint32_t rot, uint32_t pos0, + float theta, int gptj) { + if (!ctx().ok || !ctx().psoRopeFull) return false; + ensureEncoder(); + uint32_t gj = gptj ? 1u : 0u; + [g_enc setComputePipelineState:ctx().psoRopeFull]; + setBufAt(in, (size_t)tokens*D*2, 0); setBufAt(out, (size_t)tokens*D*2, 1); + [g_enc setBytes:&S length:4 atIndex:2]; [g_enc setBytes:&H length:4 atIndex:3]; + [g_enc setBytes:&D length:4 atIndex:4]; [g_enc setBytes:&rot length:4 atIndex:5]; + [g_enc setBytes:&pos0 length:4 atIndex:6]; [g_enc setBytes:&theta length:4 atIndex:7]; + [g_enc setBytes:&gj length:4 atIndex:8]; + [g_enc dispatchThreads:MTLSizeMake(D, tokens, 1) threadsPerThreadgroup:MTLSizeMake(std::min(D, 64u), 4, 1)]; + return true; +} +bool cactus_metal_encode_maxpool1d(void* out, const void* in, uint32_t NC, uint32_t L, + uint32_t Lout, uint32_t K, uint32_t stride) { + if (!ctx().ok || !ctx().psoMaxpool1d) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoMaxpool1d]; + setBufAt(in, (size_t)NC*L*2, 0); setBufAt(out, (size_t)NC*Lout*2, 1); + [g_enc setBytes:&L length:4 atIndex:2]; [g_enc setBytes:&Lout length:4 atIndex:3]; + [g_enc setBytes:&K length:4 atIndex:4]; [g_enc setBytes:&stride length:4 atIndex:5]; + [g_enc dispatchThreads:MTLSizeMake(Lout, NC, 1) threadsPerThreadgroup:MTLSizeMake(std::min(Lout, 64u), 4, 1)]; + return true; +} +bool cactus_metal_encode_bilinear(void* out, const void* in, uint32_t sh, uint32_t sw, + uint32_t dh, uint32_t dw, uint32_t E, int align) { + if (!ctx().ok || !ctx().psoBilinear) return false; + ensureEncoder(); + uint32_t al = align ? 1u : 0u; + [g_enc setComputePipelineState:ctx().psoBilinear]; + setBufAt(in, (size_t)sh*sw*E*2, 0); setBufAt(out, (size_t)dh*dw*E*2, 1); + [g_enc setBytes:&sh length:4 atIndex:2]; [g_enc setBytes:&sw length:4 atIndex:3]; + [g_enc setBytes:&dh length:4 atIndex:4]; [g_enc setBytes:&dw length:4 atIndex:5]; + [g_enc setBytes:&E length:4 atIndex:6]; [g_enc setBytes:&al length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake(E, dh*dw, 1) threadsPerThreadgroup:MTLSizeMake(std::min(E, 64u), 4, 1)]; + return true; +} +bool cactus_metal_encode_conv1d_gen(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t Cin, uint32_t L, uint32_t Cout, + uint32_t Lout, uint32_t K, uint32_t stride, int w_ck_co) { + if (!ctx().ok || !ctx().psoConv1dGen) return false; + ensureEncoder(); + uint32_t hb = bias ? 1u : 0u, wl = w_ck_co ? 1u : 0u; + [g_enc setComputePipelineState:ctx().psoConv1dGen]; + setBufAt(x, (size_t)N*Cin*L*2, 0); setBufAt(w, (size_t)Cout*Cin*K*2, 1); + setBufAt(bias ? bias : x, bias ? (size_t)Cout*2 : 2, 2); + setBufAt(out, (size_t)N*Cout*Lout*2, 3); + [g_enc setBytes:&Cin length:4 atIndex:4]; [g_enc setBytes:&L length:4 atIndex:5]; + [g_enc setBytes:&Cout length:4 atIndex:6]; [g_enc setBytes:&Lout length:4 atIndex:7]; + [g_enc setBytes:&K length:4 atIndex:8]; [g_enc setBytes:&stride length:4 atIndex:9]; + [g_enc setBytes:&hb length:4 atIndex:10]; [g_enc setBytes:&wl length:4 atIndex:11]; + [g_enc dispatchThreads:MTLSizeMake(Lout, Cout, N) threadsPerThreadgroup:MTLSizeMake(std::min(Lout, 64u), 4, 1)]; + return true; +} +bool cactus_metal_encode_conv1d_nlc_dw(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t L, uint32_t C, uint32_t K, + uint32_t dil, uint32_t pad) { + if (!ctx().ok || !ctx().psoConv1dNlcDw) return false; + ensureEncoder(); + uint32_t hb = bias ? 1u : 0u; + [g_enc setComputePipelineState:ctx().psoConv1dNlcDw]; + setBufAt(x, (size_t)N*L*C*2, 0); setBufAt(w, (size_t)C*K*2, 1); + setBufAt(bias ? bias : x, bias ? (size_t)C*2 : 2, 2); + setBufAt(out, (size_t)N*L*C*2, 3); + [g_enc setBytes:&L length:4 atIndex:4]; [g_enc setBytes:&C length:4 atIndex:5]; + [g_enc setBytes:&K length:4 atIndex:6]; [g_enc setBytes:&dil length:4 atIndex:7]; + [g_enc setBytes:&pad length:4 atIndex:8]; [g_enc setBytes:&hb length:4 atIndex:9]; + [g_enc dispatchThreads:MTLSizeMake(C, L, N) threadsPerThreadgroup:MTLSizeMake(std::min(C, 64u), 4, 1)]; + return true; +} +bool cactus_metal_encode_conv2d(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t Cin, uint32_t H, uint32_t W, uint32_t Cout, + uint32_t Ho, uint32_t Wo, uint32_t K, uint32_t stride, + uint32_t pad, int dw) { + if (!ctx().ok || !ctx().psoConv2d) return false; + ensureEncoder(); + uint32_t hb = bias ? 1u : 0u, dwf = dw ? 1u : 0u; + size_t wbytes = dw ? (size_t)Cout*K*K*2 : (size_t)Cout*Cin*K*K*2; + [g_enc setComputePipelineState:ctx().psoConv2d]; + setBufAt(x, (size_t)N*Cin*H*W*2, 0); setBufAt(w, wbytes, 1); + setBufAt(bias ? bias : x, bias ? (size_t)Cout*2 : 2, 2); + setBufAt(out, (size_t)N*Cout*Ho*Wo*2, 3); + [g_enc setBytes:&Cin length:4 atIndex:4]; [g_enc setBytes:&H length:4 atIndex:5]; + [g_enc setBytes:&W length:4 atIndex:6]; [g_enc setBytes:&Cout length:4 atIndex:7]; + [g_enc setBytes:&Ho length:4 atIndex:8]; [g_enc setBytes:&Wo length:4 atIndex:9]; + [g_enc setBytes:&K length:4 atIndex:10]; [g_enc setBytes:&stride length:4 atIndex:11]; + [g_enc setBytes:&pad length:4 atIndex:12]; [g_enc setBytes:&dwf length:4 atIndex:13]; + [g_enc setBytes:&hb length:4 atIndex:14]; + [g_enc dispatchThreads:MTLSizeMake(Wo, Ho, (size_t)N*Cout) threadsPerThreadgroup:MTLSizeMake(std::min(Wo, 32u), std::min(Ho, 8u), 1)]; + return true; +} +bool cactus_metal_encode_batchnorm(void* out, const void* x, const void* w, const void* b, + const void* rm, const void* rv, uint32_t C, uint32_t inner, + uint32_t total, float eps) { + if (!ctx().ok || !ctx().psoBatchnorm) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoBatchnorm]; + setBufAt(x, (size_t)total*2, 0); setBufAt(w, (size_t)C*2, 1); setBufAt(b, (size_t)C*2, 2); + setBufAt(rm, (size_t)C*2, 3); setBufAt(rv, (size_t)C*2, 4); setBufAt(out, (size_t)total*2, 5); + [g_enc setBytes:&C length:4 atIndex:6]; [g_enc setBytes:&inner length:4 atIndex:7]; + [g_enc setBytes:&eps length:4 atIndex:8]; [g_enc setBytes:&total length:4 atIndex:9]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_groupnorm(void* out, const void* x, const void* w, const void* b, + uint32_t N, uint32_t C, uint32_t S, uint32_t groups, float eps) { + if (!ctx().ok || !ctx().psoGroupnorm || groups == 0 || C % groups != 0) return false; + ensureEncoder(); + uint32_t cpg = C / groups; + [g_enc setComputePipelineState:ctx().psoGroupnorm]; + setBufAt(x, (size_t)N*C*S*2, 0); setBufAt(w, (size_t)C*2, 1); setBufAt(b, (size_t)C*2, 2); + setBufAt(out, (size_t)N*C*S*2, 3); + [g_enc setBytes:&cpg length:4 atIndex:4]; [g_enc setBytes:&S length:4 atIndex:5]; + [g_enc setBytes:&C length:4 atIndex:6]; [g_enc setBytes:&eps length:4 atIndex:7]; + [g_enc setThreadgroupMemoryLength:256*4 atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(groups, N, 1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_bias_add_rows(void* y, const void* bias, uint32_t C, uint32_t total) { + if (!ctx().ok || !ctx().psoBiasAddRows) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoBiasAddRows]; + setBufAt(y, (size_t)total*2, 0); setBufAt(bias, (size_t)C*2, 1); + [g_enc setBytes:&C length:4 atIndex:2]; [g_enc setBytes:&total length:4 atIndex:3]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a) { if (!ctx().ok || !ctx().psoGemmBatch || !ctx().psoGemmBatchF32A) return false; diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 5e62e391b..025e8f942 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -74,6 +74,19 @@ bool cactus_metal_encode_gemm_f16(void*, const void*, const void*, uint32_t, uin bool cactus_metal_encode_attention_f16(void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, uint32_t, uint32_t, float, uint32_t) { return false; } +bool cactus_metal_encode_reduce_axis(int, void*, const void*, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_cumsum(void*, const void*, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_concat2(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_gather_f32idx(void*, const void*, const void*, uint32_t, uint32_t, size_t) { return false; } +bool cactus_metal_encode_rope_full(void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, int) { return false; } +bool cactus_metal_encode_maxpool1d(void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_bilinear(void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_conv1d_gen(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_conv1d_nlc_dw(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_conv2d(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_batchnorm(void*, const void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_groupnorm(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_bias_add_rows(void*, const void*, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_gemm_batch(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, int, int) { return false; } bool cactus_metal_encode_conv1d_dw(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_strided_copy(void*, const void*, const uint32_t*, const uint32_t*, From 0dc86311698506a52ba60de422b90b90d79cc50b Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 21:42:24 +0200 Subject: [PATCH 17/36] Generic elementwise chain fusion for the Metal plan compiler Rule 11 fuses maximal single-consumer chains of activations, scalar ops, same-shape binaries, and clamps into one elemwise_chain_f16 dispatch with up to eight steps and three side inputs. Values round to fp16 between steps so fused output is bit-identical to the unfused per-node path. Specialized rules keep priority; chains only claim nodes no other rule clustered. gemma-3n audio tower 181ms -> 170ms, vision 376ms -> 354ms; decode unchanged; graph, parity, and engine suites pass. Signed-off-by: jakmro Signed-off-by: jakmro --- cactus-graph/src/metal_plan.cpp | 136 ++++++++++++++++++++++ cactus-kernels/metal_backend.h | 3 + cactus-kernels/src/cactus_kernels.metal | 39 +++++++ cactus-kernels/src/metal_backend.mm | 21 +++- cactus-kernels/src/metal_backend_stub.cpp | 1 + 5 files changed, 198 insertions(+), 2 deletions(-) diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index 309a97d34..21c75cdbe 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -54,6 +54,14 @@ struct MetalFusePlan { std::vector exec_list; std::vector arena_off; char* arena_base = nullptr; + std::vector> blobs; +}; + +struct EwChainStep { + int32_t kind; + int32_t code; + float p0; + float p1; }; MetalFusePlan* cactus_metal_plan_build( @@ -871,6 +879,119 @@ MetalFusePlan* cactus_metal_plan_build( break; } + { + auto ew_kind = [&](const GraphNode& nd, int* kind, int* code, float* p0, float* p1) -> bool { + if (nd.output_buffer.precision != Precision::FP16) return false; + *p0 = 0; *p1 = 0; + switch (nd.op_type) { + case OpType::GELU: *kind = 0; *code = 0; return true; + case OpType::TANH: *kind = 0; *code = 1; return true; + case OpType::SILU: *kind = 0; *code = 2; return true; + case OpType::RELU: *kind = 0; *code = 3; return true; + case OpType::GELU_ERF: *kind = 0; *code = 4; return true; + case OpType::SIGMOID: *kind = 0; *code = 5; return true; + case OpType::SCALAR_ADD: *kind = 1; *code = 0; *p0 = nd.params.scalar; return true; + case OpType::SCALAR_SUBTRACT: *kind = 1; *code = 1; *p0 = nd.params.scalar; return true; + case OpType::SCALAR_MULTIPLY: *kind = 1; *code = 2; *p0 = nd.params.scalar; return true; + case OpType::SCALAR_DIVIDE: *kind = 1; *code = 3; *p0 = nd.params.scalar; return true; + case OpType::SCALAR_EXP: *kind = 1; *code = 4; return true; + case OpType::SCALAR_SQRT: *kind = 1; *code = 5; return true; + case OpType::SCALAR_COS: *kind = 1; *code = 6; return true; + case OpType::SCALAR_SIN: *kind = 1; *code = 7; return true; + case OpType::SCALAR_LOG: *kind = 1; *code = 8; return true; + case OpType::ABS: *kind = 1; *code = 9; return true; + case OpType::POW: *kind = 1; *code = 10; *p0 = nd.params.scalar; return true; + case OpType::SCALAR_NOT_EQUAL: *kind = 1; *code = 11; *p0 = nd.params.scalar; return true; + case OpType::LEAKY_RELU: *kind = 1; *code = 12; *p0 = nd.params.scalar; return true; + case OpType::ADD: *kind = 2; *code = 0; return true; + case OpType::ADD_CLIPPED: *kind = 2; *code = 1; return true; + case OpType::SUBTRACT: *kind = 2; *code = 2; return true; + case OpType::MULTIPLY: *kind = 2; *code = 3; return true; + case OpType::DIVIDE: *kind = 2; *code = 4; return true; + case OpType::NOT_EQUAL: *kind = 2; *code = 5; return true; + case OpType::CLAMP: *kind = 3; *code = 0; *p0 = nd.params.scalar; *p1 = nd.params.scale; return true; + default: return false; + } + }; + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + int kind, code; float p0, p1; + if (!ew_kind(*nodes[i], &kind, &code, &p0, &p1)) continue; + size_t total = nodes[i]->output_buffer.total_size; + std::vector chain{i}; + size_t cur = i; + while (chain.size() < 8) { + if (cons[cur].size() != 1) break; + size_t nxt = cons[cur][0]; + if (plan->action[nxt] != -1) break; + int k2, c2; float q0, q1; + if (!ew_kind(*nodes[nxt], &k2, &c2, &q0, &q1)) break; + if (nodes[nxt]->output_buffer.total_size != total) break; + bool feeds = false; + for (size_t a = 0; a < nodes[nxt]->input_ids.size() && a < 2; ++a) + if (idxof(nodes[nxt]->input_ids[a]) == (long)cur) feeds = true; + if (!feeds) break; + chain.push_back(nxt); + cur = nxt; + } + if (chain.size() < 2) continue; + std::vector blob; + std::vector sides; + bool ok = true; + long head_in = -1; + for (size_t ci = 0; ci < chain.size() && ok; ++ci) { + const GraphNode& cn = *nodes[chain[ci]]; + int k2, c2; float q0, q1; + ew_kind(cn, &k2, &c2, &q0, &q1); + long prev = ci == 0 ? -1 : (long)chain[ci - 1]; + if (k2 == 2) { + long in0 = idxof(cn.input_ids[0]); + long in1 = idxof(cn.input_ids[1]); + long side, chain_in; + bool rhs; + if (ci > 0 && in0 == prev) { chain_in = in0; side = in1; rhs = false; } + else if (ci > 0 && in1 == prev) { chain_in = in1; side = in0; rhs = true; } + else { chain_in = in0; side = in1; rhs = false; } + if (side < 0 || chain_in < 0 + || nodes[(size_t)side]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)side]->output_buffer.total_size != total + || plan->action[(size_t)side] == -2) { ok = false; break; } + size_t slot = sides.size(); + for (size_t si = 0; si < sides.size(); ++si) + if (sides[si] == (size_t)side) slot = si; + if (slot == sides.size()) { + if (sides.size() >= 3) { ok = false; break; } + sides.push_back((size_t)side); + } + c2 |= (rhs ? 16 : 0) | ((int)slot << 5); + if (ci == 0) head_in = chain_in; + } else if (ci == 0) { + head_in = idxof(cn.input_ids[0]); + } + EwChainStep st{k2, c2, q0, q1}; + float packed[4]; + std::memcpy(packed, &st, sizeof(st)); + blob.insert(blob.end(), packed, packed + 4); + } + if (!ok || head_in < 0 + || nodes[(size_t)head_in]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)head_in]->output_buffer.total_size != total) continue; + MetalCluster c; + c.rule = 11; + c.a0 = (size_t)head_in; + c.a1 = sides.size() > 0 ? sides[0] : (size_t)0; + c.a2 = sides.size() > 1 ? sides[1] : (size_t)0; + c.a3 = sides.size() > 2 ? sides[2] : (size_t)0; + c.u0 = (uint32_t)chain.size(); + c.u1 = (uint32_t)sides.size(); + c.b0 = plan->blobs.size(); + c.b4 = chain.back(); + plan->blobs.push_back(std::move(blob)); + std::vector cover(chain.begin(), chain.end() - 1); + add_cluster(c, chain.back(), cover); + } + } + plan->exec_list.reserve(n); for (size_t i = 0; i < n; ++i) if (plan->action[i] != -2 && nodes[i]->op_type != OpType::INPUT) @@ -1222,6 +1343,21 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, return cactus_metal_encode_strided_copy(out, pp + c.a2 * 2, oshape, sstride, ndim + 1, (uint32_t)total, 0, nodes[c.a0]->output_buffer.byte_size - c.a2 * 2, anchor.output_buffer.byte_size); } + case 11: { + GraphNode& anchor = *nodes[c.b4]; + const void* in = nodes[c.a0]->output_buffer.get_data(); + void* out = anchor.output_buffer.get_data(); + if (!in || !out) return false; + const void* sides[3] = { nullptr, nullptr, nullptr }; + size_t side_idx[3] = { c.a1, c.a2, c.a3 }; + for (uint32_t si = 0; si < c.u1; ++si) { + sides[si] = nodes[side_idx[si]]->output_buffer.get_data(); + if (!sides[si]) return false; + } + const auto& blob = p->blobs[c.b0]; + return cactus_metal_encode_elemwise_chain(out, in, blob.data(), c.u0, + sides[0], sides[1], sides[2], anchor.output_buffer.total_size); + } default: return false; } } diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 627ce932e..9fa28cee6 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -148,6 +148,9 @@ bool cactus_metal_encode_batchnorm(void* out, const void* x, const void* w, cons bool cactus_metal_encode_groupnorm(void* out, const void* x, const void* w, const void* b, uint32_t N, uint32_t C, uint32_t S, uint32_t groups, float eps); bool cactus_metal_encode_bias_add_rows(void* y, const void* bias, uint32_t C, uint32_t total); +bool cactus_metal_encode_elemwise_chain(void* out, const void* in, const float* steps, + uint32_t nsteps, const void* side0, const void* side1, + const void* side2, size_t n); bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a); bool cactus_metal_encode_conv1d_dw(void* out, const void* x, const void* w, diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index a627f7b91..6648735c5 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -1891,6 +1891,45 @@ kernel void bias_add_rows_f16(device half* y [[buffer(0)]], device const half* b if (i < n) y[i] = (half)((float)y[i] + (float)bias[i % C]); } +struct EwStep { int kind; int code; float p0; float p1; }; + +kernel void elemwise_chain_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant EwStep* steps [[buffer(2)]], constant uint& nsteps [[buffer(3)]], + device const half* s0 [[buffer(4)]], device const half* s1 [[buffer(5)]], + device const half* s2 [[buffer(6)]], constant uint& n [[buffer(7)]], + uint i [[thread_position_in_grid]]) { + if (i >= n) return; + float x = (float)in[i]; + for (uint s = 0; s < nsteps; ++s) { + EwStep st = steps[s]; + if (st.kind == 0) { + if (st.code == 0) x = gelu_tanh(x); + else if (st.code == 1) x = precise::tanh(x); + else if (st.code == 2) x = x/(1.0f+precise::exp(-x)); + else if (st.code == 4) x = 0.5f*x*(1.0f+erf_approx(x*0.70710678f)); + else if (st.code == 5) x = 1.0f/(1.0f+precise::exp(-x)); + else x = max(x, 0.0f); + } else if (st.kind == 1) { + x = scalar_apply(x, st.p0, st.code); + } else if (st.kind == 2) { + uint slot = st.code >> 5; + int op = st.code & 15; + bool rhs = (st.code & 16) != 0; + float o = (float)(slot == 0 ? s0[i] : slot == 1 ? s1[i] : s2[i]); + float a = rhs ? o : x, b = rhs ? x : o; + float r; + switch (op) { case 2: r=a-b; break; case 3: r=a*b; break; case 4: r=a/b; break; + case 5: r=(a!=b)?1.0f:0.0f; break; default: r=a+b; } + if (op == 1) r = clamp(r, -65500.0f, 65500.0f); + x = r; + } else { + x = clamp(x, st.p0, st.p1); + } + x = (float)(half)x; + } + y[i] = (half)x; +} + kernel void bcast_binary_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], device half* out [[buffer(2)]], constant uint* oshape [[buffer(3)]], constant uint* astride [[buffer(4)]], constant uint* bstride [[buffer(5)]], diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 6444eb5e5..0357f5b3d 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -54,7 +54,7 @@ id psoReduceF16=nil, psoReduceF32=nil, psoCumsumF16=nil, psoCumsumF32=nil; id psoConcat2=nil, psoGatherF32Idx=nil, psoRopeFull=nil, psoMaxpool1d=nil; id psoBilinear=nil, psoConv1dGen=nil, psoConv1dNlcDw=nil, psoConv2d=nil; - id psoBatchnorm=nil, psoGroupnorm=nil, psoBiasAddRows=nil; + id psoBatchnorm=nil, psoGroupnorm=nil, psoBiasAddRows=nil, psoEwChain=nil; id dummy=nil; bool ok = false; @@ -109,7 +109,7 @@ psoBilinear=pso("bilinear_f16"); psoConv1dGen=pso("conv1d_gen_f16"); psoConv1dNlcDw=pso("conv1d_nlc_dw_f16"); psoConv2d=pso("conv2d_f16"); psoBatchnorm=pso("batchnorm_f16"); psoGroupnorm=pso("groupnorm_f16"); - psoBiasAddRows=pso("bias_add_rows_f16"); + psoBiasAddRows=pso("bias_add_rows_f16"); psoEwChain=pso("elemwise_chain_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; @@ -1699,6 +1699,23 @@ bool cactus_metal_encode_bias_add_rows(void* y, const void* bias, uint32_t C, ui [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } +bool cactus_metal_encode_elemwise_chain(void* out, const void* in, const float* steps, + uint32_t nsteps, const void* side0, const void* side1, + const void* side2, size_t n) { + if (!ctx().ok || !ctx().psoEwChain || nsteps == 0 || nsteps > 8) return false; + ensureEncoder(); + uint32_t ns = nsteps, nn = (uint32_t)n; + [g_enc setComputePipelineState:ctx().psoEwChain]; + setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); + [g_enc setBytes:steps length:nsteps*16 atIndex:2]; + [g_enc setBytes:&ns length:4 atIndex:3]; + setBufAt(side0 ? side0 : in, n*2, 4); + setBufAt(side1 ? side1 : in, n*2, 5); + setBufAt(side2 ? side2 : in, n*2, 6); + [g_enc setBytes:&nn length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a) { if (!ctx().ok || !ctx().psoGemmBatch || !ctx().psoGemmBatchF32A) return false; diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 025e8f942..a58135d9f 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -87,6 +87,7 @@ bool cactus_metal_encode_conv2d(void*, const void*, const void*, const void*, ui bool cactus_metal_encode_batchnorm(void*, const void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, float) { return false; } bool cactus_metal_encode_groupnorm(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, float) { return false; } bool cactus_metal_encode_bias_add_rows(void*, const void*, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_elemwise_chain(void*, const void*, const float*, uint32_t, const void*, const void*, const void*, size_t) { return false; } bool cactus_metal_encode_gemm_batch(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, int, int) { return false; } bool cactus_metal_encode_conv1d_dw(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_strided_copy(void*, const void*, const uint32_t*, const uint32_t*, From ed6639127a9968c57e0c3f409246fc2eb350b785 Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 21:57:33 +0200 Subject: [PATCH 18/36] Style and DRY pass over the Metal branch - remove dead code: rms_norm_rope encode path and kernel, unused gsc/smul locals, sampling_needs_adjust and stale step flags in metal_runtime - deduplicate kv-append encode variants behind shared static helpers and the persistent-activation allocation into one lambda in the executor - guard ATTENTION_CACHED plan matching against unmapped query input; add ctx/W guards to quant matmul encodes for consistency - free the ortho scratch buffer before regrowing; rename a local shadowing the buffer helper; drop an inverted empty branch in the CAT rules - update stale --backend help text that claimed gemma-only Metal support Signed-off-by: jakmro Signed-off-by: jakmro --- cactus-graph/src/execute.cpp | 42 +++++------ cactus-graph/src/metal_plan.cpp | 11 ++- cactus-graph/src/metal_runtime.cpp | 6 -- cactus-kernels/metal_backend.h | 3 - cactus-kernels/src/cactus_kernels.metal | 30 -------- cactus-kernels/src/metal_backend.mm | 86 ++++++++++------------- cactus-kernels/src/metal_backend_stub.cpp | 1 - python/cactus/cli/__init__.py | 8 +-- 8 files changed, 64 insertions(+), 123 deletions(-) diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index d3abf88af..8683ac8aa 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -1299,6 +1299,20 @@ void CactusGraph::execute(const std::string& profile_file) { GraphNode& nd = *nodes_[idx]; if (aliases_input(nd)) nd.output_buffer.external_data = nullptr; }; + auto assign_persistent_act = [&](GraphNode& nd) { + size_t need = nd.output_buffer.byte_size; + auto pit = metal_persistent_acts_.find(nd.id); + void* p = nullptr; + if (pit != metal_persistent_acts_.end() && pit->second.second >= need) { + p = pit->second.first; + } else { + if (pit != metal_persistent_acts_.end()) cactus_metal_free_shared(pit->second.first); + p = cactus_metal_alloc_pooled(need); + if (p) metal_persistent_acts_[nd.id] = { p, need }; + } + if (p) { nd.output_buffer.release_to_pool(pool); nd.output_buffer.set_external(p); } + else nd.output_buffer.resize_from_pool(pool); + }; size_t since_flush = 0; if (fplan) cactus_metal_plan_fold(fplan, nodes_); std::vector metal_live(n, 0); @@ -1329,17 +1343,7 @@ void CactusGraph::execute(const std::string& profile_file) { continue; } if (fact == -3) { - size_t need = node->output_buffer.byte_size; - auto pit = metal_persistent_acts_.find(node->id); - void* p = nullptr; - if (pit != metal_persistent_acts_.end() && pit->second.second >= need) p = pit->second.first; - else { - if (pit != metal_persistent_acts_.end()) cactus_metal_free_shared(pit->second.first); - p = cactus_metal_alloc_pooled(need); - if (p) metal_persistent_acts_[node->id] = { p, need }; - } - if (p) { node->output_buffer.release_to_pool(pool); node->output_buffer.set_external(p); } - else node->output_buffer.resize_from_pool(pool); + assign_persistent_act(*node); metal_live[i] = 1; for (size_t r : release_after[i]) metal_release(r); continue; @@ -1361,18 +1365,7 @@ void CactusGraph::execute(const std::string& profile_file) { node->output_buffer.release_to_pool(pool); node->output_buffer.set_external(ap); } else { - size_t need = node->output_buffer.byte_size; - auto pit = metal_persistent_acts_.find(node->id); - void* p = nullptr; - if (pit != metal_persistent_acts_.end() && pit->second.second >= need) { - p = pit->second.first; - } else { - if (pit != metal_persistent_acts_.end()) cactus_metal_free_shared(pit->second.first); - p = cactus_metal_alloc_pooled(need); - if (p) metal_persistent_acts_[node->id] = { p, need }; - } - if (p) { node->output_buffer.release_to_pool(pool); node->output_buffer.set_external(p); } - else node->output_buffer.resize_from_pool(pool); + assign_persistent_act(*node); } } if (fact >= 0) { @@ -1924,8 +1917,7 @@ void CactusGraph::soft_reset_keep_pool() { } void CactusGraph::prewarm_metal_quant_weights() { - const bool gpu = cactus_backend_metal(); - if (!gpu) return; + if (!cactus_backend_metal()) return; for (auto& np : nodes_) { GraphNode& node = *np; if (node.op_type != OpType::MATMUL || node.input_ids.size() < 2) continue; diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index 21c75cdbe..fb188b064 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -241,7 +241,9 @@ MetalFusePlan* cactus_metal_plan_build( GraphNode& nd = *nodes[i]; if (nd.op_type == OpType::ATTENTION_CACHED && nd.input_ids.size() >= 5) { - const BufferDesc& qb = nodes[(size_t)idxof(nd.input_ids[0])]->output_buffer; + long qidx = idxof(nd.input_ids[0]); + if (qidx < 0) continue; + const BufferDesc& qb = nodes[(size_t)qidx]->output_buffer; if (qb.shape.size() < 4 || qb.shape[0] != 1 || qb.shape[1] != 1) continue; uint32_t nqh = (uint32_t)qb.shape[2], hd = (uint32_t)qb.shape[3]; long qk = idxof(nd.input_ids[1]); @@ -340,8 +342,6 @@ MetalFusePlan* cactus_metal_plan_build( long mul = up_in(i, 0); if (mul >= 0 && nodes[(size_t)mul]->op_type == OpType::MULTIPLY && sole_use(mul, {i})) { long g0 = up_in((size_t)mul, 0), u0v = up_in((size_t)mul, 1); - float gsc = 1.0f; - long smul = -1; for (int side = 0; side < 2; ++side) { long gel = side == 0 ? g0 : u0v; long other = side == 0 ? u0v : g0; @@ -414,11 +414,9 @@ MetalFusePlan* cactus_metal_plan_build( if (sm >= 0) cover.push_back((size_t)sm); collect_chain((size_t)idxof(nd.input_ids[0]), mul, cover); add_cluster(c, i, cover); - gsc = sc_val; smul = sm; break; } } - (void)gsc; (void)smul; if (plan->action[i] >= 0) continue; } } @@ -480,8 +478,7 @@ MetalFusePlan* cactus_metal_plan_build( if (ip * cnt != op2) blocky = false; } else blocky = false; } - if (!blocky) { } - else { + if (blocky) { long m0 = -1; { long t = up_in(i, 0); diff --git a/cactus-graph/src/metal_runtime.cpp b/cactus-graph/src/metal_runtime.cpp index ba02d0174..4ed7ee245 100644 --- a/cactus-graph/src/metal_runtime.cpp +++ b/cactus-graph/src/metal_runtime.cpp @@ -40,8 +40,6 @@ struct GSampling { long long suppressed = -1; }; static GSampling g_samp; -static bool g_step_adjusted = false; -static bool g_step_amax_biased = false; static float* g_bias_dense = nullptr; static size_t g_bias_dense_n = 0; @@ -111,10 +109,6 @@ void cactus_graph_metal_tail_commit() { g_last_amax_biased = g_plan_tail_biased; } -static inline bool sampling_needs_adjust() { - return g_samp.active && - ((g_samp.rep_penalty != 1.0f && !g_samp.recent.empty()) || g_samp.suppressed >= 0); -} bool cactus_graph_metal_argmax(uint32_t* idx, float* best, float* second) { if (!g_metal_argmax_valid || !g_metal_argmax_buf) return false; diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 9fa28cee6..00a1e7c98 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -43,9 +43,6 @@ bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const v uint32_t heads, uint32_t head_dim); bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps); -bool cactus_metal_encode_rms_norm_rope(void* out, const void* x, const void* weight, - const void* cos, const void* sin, - uint32_t heads, uint32_t head_dim, float eps); bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* weight, const void* res, size_t rows, size_t dim, float eps); bool cactus_metal_encode_rms_norm_add_rms(void* h_out, void* xn_out, const void* in, const void* w1, diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 6648735c5..1d3712317 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -947,36 +947,6 @@ kernel void rope_f16(device const half* x [[buffer(0)]], device half* out [[buff } -kernel void rms_norm_rope_f16( - device const half* x [[buffer(0)]], - device const half* w [[buffer(1)]], - device const half* cs [[buffer(2)]], - device const half* sn [[buffer(3)]], - device half* out [[buffer(4)]], - constant uint& hd [[buffer(5)]], - constant float& eps [[buffer(6)]], - uint head [[threadgroup_position_in_grid]], - uint t [[thread_position_in_threadgroup]], - uint nt [[threads_per_threadgroup]], - threadgroup float* red [[threadgroup(0)]]) -{ - device const half* xh = x + (size_t)head*hd; - device half* oh = out + (size_t)head*hd; - float partial=0; - for (uint i=t;i0; s>>=1){ if (tgroup_size, K=W->K, N=W->N, ng=W->num_groups; if (ctx().ok && (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->rotation && ng==1 && gs==K && (N%4)==0) { static void* ortho_code = nullptr; static uint32_t ortho_code_k = 0; - if (ortho_code_k < K) { ortho_code = cactus_metal_alloc_shared((size_t)K*sizeof(__fp16)); ortho_code_k = K; } + if (ortho_code_k < K) { + if (ortho_code) cactus_metal_free_shared(ortho_code); + ortho_code = cactus_metal_alloc_shared((size_t)K*sizeof(__fp16)); + ortho_code_k = K; + } if (ortho_code) return cactus_metal_encode_quant_matmul_ortho(out, lhs, ortho_code, W); } const bool fast = ctx().ok && quant_fast_eligible(W); @@ -823,6 +828,7 @@ bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { } bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M) { + if (!ctx().ok || !W) return false; const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; const uint32_t bits=W->bits, pgb=(gs*bits+7u)/8u; @@ -1327,9 +1333,9 @@ bool cactus_metal_encode_conv1d_k3(void* out, const void* x, const void* w, int auto it = g_dequant_cache.find(w); if (it == g_dequant_cache.end()) { size_t rows = Cout, cols = (size_t)Cin*3; - id buf = [ctx().dev newBufferWithLength:rows*cols*2 options:MTLResourceStorageModeShared]; - if (!buf) return false; - __fp16* dst = (__fp16*)[buf contents]; + id dq = [ctx().dev newBufferWithLength:rows*cols*2 options:MTLResourceStorageModeShared]; + if (!dq) return false; + __fp16* dst = (__fp16*)[dq contents]; const int8_t* src = (const int8_t*)w; if (w_gs == 0 || !w_scales) { for (size_t i = 0; i < rows*cols; ++i) dst[i] = (__fp16)src[i]; @@ -1340,7 +1346,7 @@ bool cactus_metal_encode_conv1d_k3(void* out, const void* x, const void* w, int for (size_t c = 0; c < cols; ++c) dst[r*cols+c] = (__fp16)((float)src[r*cols+c] * (float)scales[r*ngr + c/w_gs]); } - it = g_dequant_cache.emplace(w, buf).first; + it = g_dequant_cache.emplace(w, dq).first; } wf16 = [it->second contents]; } @@ -1788,33 +1794,33 @@ bool cactus_metal_encode_strided_scatter(void* out, const void* in, const uint32 [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } -bool cactus_metal_encode_kv_append_i8(const void* src, void* int8base, void* scalebase, - uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, +static bool encode_kv_append(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { if (!ctx().ok) return false; - uint32_t num_groups = (hdim + group_size - 1)/group_size, n = kv_heads*num_groups; + uint32_t num_groups = (hdim + group_size - 1)/group_size, n = M*kv_heads*num_groups; ensureEncoder(); - [g_enc setComputePipelineState:ctx().psoKvAppend]; + [g_enc setComputePipelineState:(M == 1 ? ctx().psoKvAppend : ctx().psoKvAppendM)]; setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; [g_enc setBytes:¤t_len length:4 atIndex:5]; [g_enc setBytes:&group_size length:4 atIndex:6]; + if (M != 1) [g_enc setBytes:&M length:4 atIndex:7]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; return true; } +bool cactus_metal_encode_kv_append_i8(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, + size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + return encode_kv_append(src, int8base, scalebase, kv_heads, hdim, current_len, group_size, 1, + src_bytes, int8_bytes, scale_bytes); +} + bool cactus_metal_encode_kv_append_i8_m(const void* src, void* int8base, void* scalebase, uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { - if (!ctx().ok) return false; - uint32_t num_groups = (hdim + group_size - 1)/group_size, n = M*kv_heads*num_groups; - ensureEncoder(); - [g_enc setComputePipelineState:ctx().psoKvAppendM]; - setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); - [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; - [g_enc setBytes:¤t_len length:4 atIndex:5]; [g_enc setBytes:&group_size length:4 atIndex:6]; - [g_enc setBytes:&M length:4 atIndex:7]; - [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; - return true; + return encode_kv_append(src, int8base, scalebase, kv_heads, hdim, current_len, group_size, M, + src_bytes, int8_bytes, scale_bytes); } bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, void* scalebase, @@ -1833,9 +1839,9 @@ bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, vo return true; } -bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, void* scalebase, +static bool encode_kv_append_sliding(const void* src, void* int8base, void* scalebase, uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, - uint32_t group_size, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + uint32_t group_size, uint32_t M, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { if (!ctx().ok) return false; uint32_t num_groups = (hdim + group_size - 1)/group_size, per = kv_heads*num_groups; id scrI8 = recycled((size_t)(remaining?remaining:1)*kv_heads*hdim); @@ -1851,44 +1857,30 @@ bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, v [g_enc setBytes:&remaining length:4 atIndex:8]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; } - uint32_t n2 = (remaining+1)*per; - [g_enc setComputePipelineState:ctx().psoSlideR]; + uint32_t n2 = (remaining+M)*per; + [g_enc setComputePipelineState:(M == 1 ? ctx().psoSlideR : ctx().psoSlideRM)]; setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); [g_enc setBuffer:scrI8 offset:0 atIndex:3]; [g_enc setBuffer:scrSc offset:0 atIndex:4]; [g_enc setBytes:&kv_heads length:4 atIndex:5]; [g_enc setBytes:&hdim length:4 atIndex:6]; [g_enc setBytes:&group_size length:4 atIndex:7]; [g_enc setBytes:&keep_sink length:4 atIndex:8]; [g_enc setBytes:&remaining length:4 atIndex:9]; + if (M != 1) [g_enc setBytes:&M length:4 atIndex:10]; [g_enc dispatchThreads:MTLSizeMake(n2,1,1) threadsPerThreadgroup:MTLSizeMake(n2<256?n2:256,1,1)]; return true; } +bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, + uint32_t group_size, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + return encode_kv_append_sliding(src, int8base, scalebase, kv_heads, hdim, keep_sink, remaining, + shift_src, group_size, 1, src_bytes, int8_bytes, scale_bytes); +} + bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, void* scalebase, uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, uint32_t group_size, uint32_t M, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { - if (!ctx().ok) return false; - uint32_t num_groups = (hdim + group_size - 1)/group_size, per = kv_heads*num_groups; - id scrI8 = recycled((size_t)(remaining?remaining:1)*kv_heads*hdim); - id scrSc = recycled((size_t)(remaining?remaining:1)*kv_heads*num_groups*sizeof(float)); - ensureEncoder(); - if (remaining > 0) { - uint32_t n = remaining*per; - [g_enc setComputePipelineState:ctx().psoSlideS]; - setBufAt(int8base, int8_bytes, 0); setBufAt(scalebase, scale_bytes, 1); - [g_enc setBuffer:scrI8 offset:0 atIndex:2]; [g_enc setBuffer:scrSc offset:0 atIndex:3]; - [g_enc setBytes:&kv_heads length:4 atIndex:4]; [g_enc setBytes:&hdim length:4 atIndex:5]; - [g_enc setBytes:&group_size length:4 atIndex:6]; [g_enc setBytes:&shift_src length:4 atIndex:7]; - [g_enc setBytes:&remaining length:4 atIndex:8]; - [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; - } - uint32_t n2 = (remaining+M)*per; - [g_enc setComputePipelineState:ctx().psoSlideRM]; - setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); - [g_enc setBuffer:scrI8 offset:0 atIndex:3]; [g_enc setBuffer:scrSc offset:0 atIndex:4]; - [g_enc setBytes:&kv_heads length:4 atIndex:5]; [g_enc setBytes:&hdim length:4 atIndex:6]; - [g_enc setBytes:&group_size length:4 atIndex:7]; [g_enc setBytes:&keep_sink length:4 atIndex:8]; - [g_enc setBytes:&remaining length:4 atIndex:9]; [g_enc setBytes:&M length:4 atIndex:10]; - [g_enc dispatchThreads:MTLSizeMake(n2,1,1) threadsPerThreadgroup:MTLSizeMake(n2<256?n2:256,1,1)]; - return true; + return encode_kv_append_sliding(src, int8base, scalebase, kv_heads, hdim, keep_sink, remaining, + shift_src, group_size, M, src_bytes, int8_bytes, scale_bytes); } void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index a58135d9f..ba69977f6 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -33,7 +33,6 @@ bool cactus_metal_encode_swiglu(void*, const void*, const void*, size_t, float) bool cactus_metal_encode_rope(void*, const void*, const void*, const void*, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_rope_m(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_rms_norm(void*, const void*, const void*, size_t, size_t, float) { return false; } -bool cactus_metal_encode_rms_norm_rope(void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, float) { return false; } bool cactus_metal_encode_rms_norm_add(void*, const void*, const void*, const void*, size_t, size_t, float) { return false; } bool cactus_metal_encode_rms_norm_add_rms(void*, void*, const void*, const void*, const void*, const void*, size_t, size_t, float, float) { return false; } diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index f9f61236b..a7a394139 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -286,7 +286,7 @@ def create_parser(): run_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, help="Maximum time to wait for cloud handoff before falling back locally") run_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", - help="Inference backend, default auto (Metal on Apple GPU, gemma-4-E2B-it only)") + help="Inference backend, default auto (Metal on Apple Silicon GPUs)") transcribe_parser = subparsers.add_parser("transcribe", help="Transcribe audio with a model", parents=[_telemetry_parent(), _build_parent()]) @@ -316,7 +316,7 @@ def create_parser(): serve_parser.add_argument("--no-access-log", action="store_true", help="Disable per-request HTTP access logging") serve_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", - help="Inference backend, default auto (Metal on Apple GPU, gemma-4-E2B-it only)") + help="Inference backend, default auto (Metal on Apple Silicon GPUs)") code_parser = subparsers.add_parser("code", help="Run the Cactus coding agent (TUI / print mode)", parents=[_build_parent()]) @@ -335,7 +335,7 @@ def create_parser(): code_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, help="Maximum time to wait for cloud handoff before falling back locally") code_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", - help="Inference backend, default auto (Metal on Apple GPU, gemma-4-E2B-it only)") + help="Inference backend, default auto (Metal on Apple Silicon GPUs)") code_parser.add_argument("agent_args", nargs=argparse.REMAINDER, help="Arguments passed through to the coding agent (prefix with -- )") @@ -344,7 +344,7 @@ def create_parser(): test_parser.add_argument("--component", choices=COMPONENTS, default="all", help="Component to test (default: all)") test_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", - help="Inference backend, default auto (Metal on Apple GPU, gemma-4-E2B-it only)") + help="Inference backend, default auto (Metal on Apple Silicon GPUs)") test_parser.add_argument("--model", dest="model_id", default=None, type=_hf_id_or_path, help=f"HF model ID under test (default: {DEFAULT_TEST_MODEL_ID})") From 6439dfd661a7eb819fbd251112e8b492725f26a4 Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 22:08:30 +0200 Subject: [PATCH 19/36] Release Metal prefill dequant cache after each completion The prefill dense-gemm path caches fp16-dequantized weights (~3.8GB for gemma-3n-e2b cq4) that decode never reads. A guard in cactus_complete now trims the cache when the turn finishes: steady-state footprint after a completion drops from ~7.3GB to 3.5GB while benchmark prefill (2060 tps @256) and decode (110 tps) are unchanged; the next turn re-dequantizes lazily during its prefill. The MPS attention scores buffer is likewise released on invalidate_host_wraps instead of living in a function-local static. Signed-off-by: jakmro Signed-off-by: jakmro --- cactus-engine/src/complete.cpp | 5 +++++ cactus-kernels/metal_backend.h | 1 + cactus-kernels/src/metal_backend.mm | 17 ++++++++++++++--- cactus-kernels/src/metal_backend_stub.cpp | 1 + 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 85273c3c9..c376a85ed 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -4,6 +4,7 @@ #include "chat_tools.h" #include "telemetry.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include "wav.h" #include #include @@ -796,6 +797,10 @@ int cactus_complete( const uint8_t* pcm_buffer, size_t pcm_buffer_size ) { + struct MetalTrimGuard { + ~MetalTrimGuard() { cactus_metal_trim_prefill_cache(); } + } metal_trim_guard; + if (!model) { std::string error_msg = last_error_message.empty() ? "Model not initialized. Check model path and files." : last_error_message; diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 00a1e7c98..465e4e943 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -28,6 +28,7 @@ void cactus_metal_session_begin(); void cactus_metal_session_sync(); void cactus_metal_session_flush(); void cactus_metal_invalidate_host_wraps(); +void cactus_metal_trim_prefill_cache(); void cactus_metal_session_end(); void* cactus_metal_alloc_shared(size_t bytes); diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index d4ca53fdd..ab84f4a6d 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -131,6 +131,8 @@ id wf16=nil; }; std::unordered_map g_resident; +bool g_wf16_live = false; +id g_attn_scores = nil; std::unordered_map g_resident_emb; std::mutex g_resident_mu; @@ -364,6 +366,7 @@ void cactus_metal_session_end() { } void cactus_metal_invalidate_host_wraps() { std::lock_guard lk(g_resident_mu); + g_attn_scores = nil; g_wrapped.clear(); g_readonly.clear(); g_resident.clear(); @@ -646,6 +649,13 @@ static inline bool quant_lowbit_eligible(const CactusQuantMatrix* W) { W->codebook && W->norms && W->packed_indices; } +void cactus_metal_trim_prefill_cache() { + if (!g_wf16_live) return; + std::lock_guard lk(g_resident_mu); + for (auto& kv : g_resident) kv.second.wf16 = nil; + g_wf16_live = false; +} + bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W) { if (!ctx().ok || !W) return false; const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; @@ -853,6 +863,7 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus if ((g_predequant && ctx().psoGdense && ctx().psoDeq) || lowbit) { if (!rw.wf16) { rw.wf16 = [ctx().dev newBufferWithLength:(size_t)N*K*2 options:MTLResourceStorageModeShared]; + g_wf16_live = true; [g_enc setComputePipelineState:(lowbit?ctx().psoDeqLow:ctx().psoDeq)]; [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.wf16 offset:0 atIndex:3]; @@ -1454,10 +1465,10 @@ bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, bool aligned = (qb.second % 16)==0 && (kb.second % 16)==0 && (vb.second % 16)==0 && (ob.second % 16)==0 && ((D*2) % 16)==0 && ((DV*2) % 16)==0; if (aligned) { - static id sbuf = nil; size_t need = (size_t)HQ*T*S*2; - if (!sbuf || sbuf.length < need) - sbuf = [ctx().dev newBufferWithLength:need options:MTLResourceStorageModeShared]; + if (!g_attn_scores || g_attn_scores.length < need) + g_attn_scores = [ctx().dev newBufferWithLength:need options:MTLResourceStorageModeShared]; + id sbuf = g_attn_scores; if (sbuf) { float beta = 0.0f; if (mask_mode == 2) { diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index ba69977f6..71adf3516 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -21,6 +21,7 @@ void cactus_metal_session_begin() {} void cactus_metal_session_sync() {} void cactus_metal_session_flush() {} void cactus_metal_invalidate_host_wraps() {} +void cactus_metal_trim_prefill_cache() {} void cactus_metal_session_end() {} void* cactus_metal_alloc_shared(size_t) { return nullptr; } void* cactus_metal_alloc_pooled(size_t) { return nullptr; } From 159244c3b14afd99feb4a61ca2e5a19b878a2585 Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 22:37:48 +0200 Subject: [PATCH 20/36] Fix MatMul CQ staleness, pin retained outputs against fusion, PR review fixes - the tq expand cache added in 3811cbc6 keyed entries by the packed-weights pointer alone; remapping different weights at the same address (as the MatMul CQ test does per bit width) served stale expansions. Entries now carry a shape-and-content fingerprint and re-expand on mismatch. Fixes the unit-test failure present on main since June 21 - Metal quant matmul eligibility now requires the 4-row interleaved layout the kernels assume; synthetic flat-layout weights fall back to CPU - plan builder takes the retained/persistent node set and refuses to cover pinned nodes, so fusion clusters can no longer swallow graph outputs - run.py reads --backend via getattr like every other CLI entry point; cactus_set_backend exported from the package root; test bundle discovery prefers the highest-quality quant when several are present Signed-off-by: jakmro Signed-off-by: jakmro --- cactus-graph/src/execute.cpp | 10 +++++++--- cactus-graph/src/metal_plan.cpp | 16 ++++++++++++---- cactus-kernels/src/matmul.cpp | 27 ++++++++++++++++++++------- cactus-kernels/src/metal_backend.mm | 21 ++++----------------- python/cactus/__init__.py | 1 + python/cactus/cli/run.py | 2 +- python/tests/bundles.py | 2 +- 7 files changed, 46 insertions(+), 33 deletions(-) diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 8683ac8aa..9befe427e 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -1052,7 +1052,8 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n struct MetalFusePlan; MetalFusePlan* cactus_metal_plan_build(const std::vector>& nodes, - const std::unordered_map& map); + const std::unordered_map& map, + const std::unordered_set& pinned_ids); void cactus_metal_plan_free(MetalFusePlan* p); void cactus_graph_metal_tail_commit(); void cactus_metal_plan_disable(MetalFusePlan* p); @@ -1222,8 +1223,11 @@ void CactusGraph::execute(const std::string& profile_file) { } } auto it = metal_plans_.find(sig); - if (it == metal_plans_.end()) - it = metal_plans_.emplace(sig, cactus_metal_plan_build(nodes_, node_index_map_)).first; + if (it == metal_plans_.end()) { + std::unordered_set pinned(retained_output_node_ids_.begin(), retained_output_node_ids_.end()); + pinned.insert(persistent_node_ids_.begin(), persistent_node_ids_.end()); + it = metal_plans_.emplace(sig, cactus_metal_plan_build(nodes_, node_index_map_, pinned)).first; + } fplan = it->second; } const uint8_t* rplan = (metal_mode && !need_debug && !metal_retype_disabled_ diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index fb188b064..140865511 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -66,11 +66,15 @@ struct EwChainStep { MetalFusePlan* cactus_metal_plan_build( const std::vector>& nodes, - const std::unordered_map& map) { + const std::unordered_map& map, + const std::unordered_set& pinned_ids) { const size_t n = nodes.size(); auto plan = new MetalFusePlan(); plan->built = true; plan->action.assign(n, -1); + std::vector pinned(n, 0); + for (size_t i = 0; i < n; ++i) + if (pinned_ids.count(nodes[i]->id)) pinned[i] = 1; auto idxof = [&](size_t id) -> long { auto it = map.find(id); @@ -220,11 +224,13 @@ MetalFusePlan* cactus_metal_plan_build( return i; }; - auto add_cluster = [&](MetalCluster c, size_t anchor, const std::vector& cover) { + auto add_cluster = [&](MetalCluster c, size_t anchor, const std::vector& cover) -> bool { + for (size_t v : cover) if (v != anchor && pinned[v]) return false; int32_t cid = (int32_t)plan->clusters.size(); plan->clusters.push_back(c); for (size_t v : cover) if (v != anchor) plan->action[v] = -2; plan->action[anchor] = cid; + return true; }; struct AttnCand { @@ -814,8 +820,8 @@ MetalFusePlan* cactus_metal_plan_build( for (uint32_t bi = 0; bi < c.u0; ++bi) if (!c.sc[bi]) c.sc[bi] = cactus_metal_alloc_shared(maxK * 2); std::vector cover; - add_cluster(c, mms[base + cnt - 1], cover); - for (size_t mi = 0; mi + 1 < cnt; ++mi) plan->action[mms[base + mi]] = -3; + if (add_cluster(c, mms[base + cnt - 1], cover)) + for (size_t mi = 0; mi + 1 < cnt; ++mi) plan->action[mms[base + mi]] = -3; } } } @@ -845,6 +851,8 @@ MetalFusePlan* cactus_metal_plan_build( } } } + for (size_t v = 0; v < n && valid; ++v) + if ((mark[v] == 1 || mark[v] == 3) && pinned[v]) valid = false; if (valid) { for (size_t v = 0; v < n; ++v) if ((mark[v] == 1 || mark[v] == 3) && plan->action[v] == -1) plan->action[v] = -2; diff --git a/cactus-kernels/src/matmul.cpp b/cactus-kernels/src/matmul.cpp index a082d3c91..2873ac911 100644 --- a/cactus-kernels/src/matmul.cpp +++ b/cactus-kernels/src/matmul.cpp @@ -1749,24 +1749,37 @@ void cactus_quant_matmul( constexpr size_t TILE_M = 8; + struct ExpandEntry { + uint64_t fingerprint = 0; + std::vector weights; + std::vector norms; + }; static std::mutex s_expand_mutex; - static std::unordered_map, std::vector>> s_expand_cache; + static std::unordered_map s_expand_cache; + const size_t packed_bytes = static_cast(W->N) * num_groups * pgb; + uint64_t fp = 1469598103934665603ull; + auto fp_mix = [&fp](uint64_t v) { fp ^= v; fp *= 1099511628211ull; }; + fp_mix(bits); fp_mix(W->K); fp_mix(W->N); fp_mix(gs); + const uint8_t* pb = static_cast(W->packed_indices); + for (size_t i = 0; i < 64 && i < packed_bytes; ++i) fp_mix(pb[i]); + for (size_t i = packed_bytes > 64 ? packed_bytes - 64 : 0; i < packed_bytes; ++i) fp_mix(pb[i]); const int8_t* w_il; const float* n_f32; { std::lock_guard lock(s_expand_mutex); auto& entry = s_expand_cache[W->packed_indices]; - if (entry.first.empty()) { + if (entry.weights.empty() || entry.fingerprint != fp) { int8_t cb_i8[16] = {}; const float cb_scale = tq_quantize_codebook_i8(W->codebook, cb_i8, codebook_size); const int8x16_t cb_lut = vld1q_s8(cb_i8); - entry.first.resize(expanded_size); - entry.second.resize(expanded_norms_size); + entry.weights.assign(expanded_size, 0); + entry.norms.assign(expanded_norms_size, 0.0f); tq_preexpand_weights(W, bits, num_groups, gs, pgb, cb_lut, cb_scale, - N_blocks, entry.first.data(), entry.second.data()); + N_blocks, entry.weights.data(), entry.norms.data()); + entry.fingerprint = fp; } - w_il = entry.first.data(); - n_f32 = entry.second.data(); + w_il = entry.weights.data(); + n_f32 = entry.norms.data(); } const size_t M_blocks = (M + TILE_M - 1) / TILE_M; diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index ab84f4a6d..a6c64d593 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -43,7 +43,7 @@ id psoAttnPre=nil, psoAttnPreMma2=nil, psoAttnPreHd256=nil; id psoKvAppendM=nil, psoKvAppendRingM=nil; id psoSlideS=nil, psoSlideR=nil, psoSlideRM=nil; - id psoRope=nil, psoRmsRope=nil; + id psoRope=nil; id psoArgmax=nil; id psoArgmaxP=nil, psoArgmaxF=nil, psoSoftcap=nil, psoAdjust=nil; id psoGather=nil; @@ -92,7 +92,7 @@ psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoAttnPreHd256=pso("attn_prefill_mma_hd256"); psoKvAppendM=pso("kv_append_i8_m"); psoKvAppendRingM=pso("kv_append_ring_i8_m"); psoSlideS=pso("kv_slide_save"); psoSlideR=pso("kv_slide_restore"); psoSlideRM=pso("kv_slide_restore_m"); - psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); psoRmsRope=pso("rms_norm_rope_f16"); + psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); psoArgmax=pso("argmax_logits"); psoArgmaxP=pso("argmax_part"); psoArgmaxF=pso("argmax_final"); psoSoftcap=pso("softcap_f16"); psoAdjust=pso("adjust_logits"); @@ -480,21 +480,6 @@ bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const v [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } -bool cactus_metal_encode_rms_norm_rope(void* out, const void* x, const void* weight, - const void* cos, const void* sin, - uint32_t heads, uint32_t head_dim, float eps) { - if (!ctx().ok || !ctx().psoRmsRope) return false; - ensureEncoder(); - uint32_t total=heads*head_dim; uint32_t hd=head_dim; float e=eps; - [g_enc setComputePipelineState:ctx().psoRmsRope]; - setBufAt(x, (size_t)total*2, 0); setBufAt(weight, (size_t)head_dim*2, 1); - setBufAt(cos, (size_t)head_dim*2, 2); setBufAt(sin, (size_t)head_dim*2, 3); - setBufAt(out, (size_t)total*2, 4); - [g_enc setBytes:&hd length:4 atIndex:5]; [g_enc setBytes:&e length:4 atIndex:6]; - [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(heads,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - return true; -} bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps) { if (!ctx().ok) return false; @@ -638,12 +623,14 @@ bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_pr } static inline bool quant_fast_eligible(const CactusQuantMatrix* W) { return W->bits == 4 && W->group_size >= 128 && (W->group_size % 128) == 0 && (W->N % 4) == 0 && + (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) && !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->input_scale_recip && W->left_signs && W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; } static inline bool quant_lowbit_eligible(const CactusQuantMatrix* W) { return (W->bits == 2 || W->bits == 3) && W->group_size == 128 && (W->N % 16) == 0 && + (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) && !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->input_scale_recip && W->left_signs && W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; diff --git a/python/cactus/__init__.py b/python/cactus/__init__.py index 3f791dd07..585fe7bf1 100644 --- a/python/cactus/__init__.py +++ b/python/cactus/__init__.py @@ -15,6 +15,7 @@ "Graph", "Tensor", "cactus_init", + "cactus_set_backend", "cactus_destroy", "cactus_reset", "cactus_stop", diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 38078a3a8..98de2d6a9 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -40,7 +40,7 @@ def cmd_run(args) -> int: ("--result-json", args.result_json), ("--confidence-threshold", args.confidence_threshold), ("--cloud-timeout-ms", args.cloud_timeout_ms), - ("--backend", args.backend), + ("--backend", getattr(args, "backend", "auto")), ): if value is not None: cmd.extend([flag, str(value)]) diff --git a/python/tests/bundles.py b/python/tests/bundles.py index 058620410..00f1548c9 100644 --- a/python/tests/bundles.py +++ b/python/tests/bundles.py @@ -27,7 +27,7 @@ def _iter_bundle_candidates(name: str): if bare.exists(): yield bare if WEIGHTS.is_dir(): - for candidate in sorted(WEIGHTS.glob(f"{name}-cq*")): + for candidate in sorted(WEIGHTS.glob(f"{name}-cq*"), reverse=True): if candidate.is_dir(): yield candidate From 25ab7ca6a5234149a2aa239b60b32c91f6e1cc85 Mon Sep 17 00:00:00 2001 From: jakmro Date: Thu, 2 Jul 2026 23:32:17 +0200 Subject: [PATCH 21/36] Flash attention kernel and kernel-level bandwidth pass for the towers - attn_flash_f16: simdgroup-matrix flash attention (online softmax, 64-query blocks, diag-rescale with skip when the running max is stable, boolean mask support); replaces the MPS decomposition for fp16 non-causal D=64 attention. Vision attention 101ms -> 71ms with no 152MB scores buffer - rms_norm_simd_f16: simdgroup-per-row reduction, four rows per threadgroup; vision norms 42ms -> 9ms - elementwise chains now span precision-cast boundaries (per-step rounding matches unfused execution) and skip retype-claimed nodes; chain, flat binary/scalar/unary/clamp (f16 and f32) kernels process four elements per thread; broadcast binary gets a row variant without per-element div chains - tiled 32x32 transpose for trailing-dim swaps, vectorized strided scatter rows for CAT, conv2d computes four outputs per thread - gemma-3n towers: audio 168ms -> 142ms, vision 353ms -> 241ms, outputs match CPU; prefill 2225 tps @256, decode unchanged; llm/vlm/stt and all graph suites pass Signed-off-by: jakmro --- cactus-graph/src/execute.cpp | 17 +- cactus-graph/src/metal_plan.cpp | 66 +++- cactus-kernels/metal_backend.h | 3 +- cactus-kernels/src/cactus_kernels.metal | 416 ++++++++++++++++++---- cactus-kernels/src/metal_backend.mm | 98 ++++- cactus-kernels/src/metal_backend_stub.cpp | 3 +- 6 files changed, 511 insertions(+), 92 deletions(-) diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 9befe427e..579707939 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -517,6 +517,15 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n const auto& perm = node.params.permutation; uint32_t nd = (uint32_t)perm.size(); if (nd == 0 || nd > 8 || in.shape.size() != nd || out.shape.size() != nd) return false; + bool tailswap = nd >= 2; + for (uint32_t d = 0; d + 2 < nd && tailswap; ++d) if (perm[d] != d) tailswap = false; + if (tailswap && (perm[nd-2] != nd-1 || perm[nd-1] != nd-2)) tailswap = false; + if (tailswap) { + size_t batch = 1; + for (uint32_t d = 0; d + 2 < nd; ++d) batch *= in.shape[d]; + return cactus_metal_encode_transpose2d(out.get_data(), in.get_data(), + (uint32_t)batch, (uint32_t)in.shape[nd-2], (uint32_t)in.shape[nd-1]); + } size_t istr[8]; row_strides(in.shape, istr); uint32_t oshape[8], sstride[8]; for (uint32_t d=0; d>& nodes, const std::unordered_map& map, - const std::unordered_set& pinned_ids); + const std::unordered_set& pinned_ids, + const std::vector& retype); void cactus_metal_plan_free(MetalFusePlan* p); void cactus_graph_metal_tail_commit(); void cactus_metal_plan_disable(MetalFusePlan* p); @@ -1226,7 +1236,10 @@ void CactusGraph::execute(const std::string& profile_file) { if (it == metal_plans_.end()) { std::unordered_set pinned(retained_output_node_ids_.begin(), retained_output_node_ids_.end()); pinned.insert(persistent_node_ids_.begin(), persistent_node_ids_.end()); - it = metal_plans_.emplace(sig, cactus_metal_plan_build(nodes_, node_index_map_, pinned)).first; + static const std::vector no_retype; + const std::vector& rt = (!metal_retype_disabled_ && metal_retype_plan_.size() == n) + ? metal_retype_plan_ : no_retype; + it = metal_plans_.emplace(sig, cactus_metal_plan_build(nodes_, node_index_map_, pinned, rt)).first; } fplan = it->second; } diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index 140865511..0ffed9e2a 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -67,7 +67,8 @@ struct EwChainStep { MetalFusePlan* cactus_metal_plan_build( const std::vector>& nodes, const std::unordered_map& map, - const std::unordered_set& pinned_ids) { + const std::unordered_set& pinned_ids, + const std::vector& retype) { const size_t n = nodes.size(); auto plan = new MetalFusePlan(); plan->built = true; @@ -886,7 +887,6 @@ MetalFusePlan* cactus_metal_plan_build( { auto ew_kind = [&](const GraphNode& nd, int* kind, int* code, float* p0, float* p1) -> bool { - if (nd.output_buffer.precision != Precision::FP16) return false; *p0 = 0; *p1 = 0; switch (nd.op_type) { case OpType::GELU: *kind = 0; *code = 0; return true; @@ -915,22 +915,30 @@ MetalFusePlan* cactus_metal_plan_build( case OpType::DIVIDE: *kind = 2; *code = 4; return true; case OpType::NOT_EQUAL: *kind = 2; *code = 5; return true; case OpType::CLAMP: *kind = 3; *code = 0; *p0 = nd.params.scalar; *p1 = nd.params.scale; return true; + case OpType::PRECISION_CAST: *kind = 4; + *code = nd.output_buffer.precision == Precision::FP32 ? 1 : 0; + return nd.output_buffer.precision == Precision::FP16 + || nd.output_buffer.precision == Precision::FP32; default: return false; } }; + auto prec_ok = [](Precision p) { return p == Precision::FP16 || p == Precision::FP32; }; + auto retyped = [&](size_t idx) { return idx < retype.size() && retype[idx] != 0; }; for (size_t i = 0; i < n; ++i) { if (plan->action[i] != -1) continue; int kind, code; float p0, p1; if (!ew_kind(*nodes[i], &kind, &code, &p0, &p1)) continue; + if (!prec_ok(nodes[i]->output_buffer.precision) || retyped(i)) continue; size_t total = nodes[i]->output_buffer.total_size; std::vector chain{i}; size_t cur = i; - while (chain.size() < 8) { + while (chain.size() < 12) { if (cons[cur].size() != 1) break; size_t nxt = cons[cur][0]; if (plan->action[nxt] != -1) break; int k2, c2; float q0, q1; if (!ew_kind(*nodes[nxt], &k2, &c2, &q0, &q1)) break; + if (!prec_ok(nodes[nxt]->output_buffer.precision) || retyped(nxt)) break; if (nodes[nxt]->output_buffer.total_size != total) break; bool feeds = false; for (size_t a = 0; a < nodes[nxt]->input_ids.size() && a < 2; ++a) @@ -939,16 +947,38 @@ MetalFusePlan* cactus_metal_plan_build( chain.push_back(nxt); cur = nxt; } - if (chain.size() < 2) continue; + size_t real_ops = 0; + for (size_t ci = 0; ci < chain.size(); ++ci) { + int k2, c2; float q0, q1; + ew_kind(*nodes[chain[ci]], &k2, &c2, &q0, &q1); + if (k2 != 4) ++real_ops; + } + if (chain.size() < 2 || real_ops == 0) continue; std::vector blob; std::vector sides; + std::vector side_f32; bool ok = true; long head_in = -1; + Precision run = Precision::FP16; for (size_t ci = 0; ci < chain.size() && ok; ++ci) { const GraphNode& cn = *nodes[chain[ci]]; int k2, c2; float q0, q1; ew_kind(cn, &k2, &c2, &q0, &q1); long prev = ci == 0 ? -1 : (long)chain[ci - 1]; + if (ci == 0) { + long hi2 = idxof(cn.input_ids[0]); + if (k2 == 2) { + long in0 = idxof(cn.input_ids[0]); + long in1 = idxof(cn.input_ids[1]); + hi2 = in0; + (void)in1; + } + if (hi2 < 0 || !prec_ok(nodes[(size_t)hi2]->output_buffer.precision) + || retyped((size_t)hi2) + || nodes[(size_t)hi2]->output_buffer.total_size != total) { ok = false; break; } + head_in = hi2; + run = nodes[(size_t)hi2]->output_buffer.precision; + } if (k2 == 2) { long in0 = idxof(cn.input_ids[0]); long in1 = idxof(cn.input_ids[1]); @@ -958,7 +988,8 @@ MetalFusePlan* cactus_metal_plan_build( else if (ci > 0 && in1 == prev) { chain_in = in1; side = in0; rhs = true; } else { chain_in = in0; side = in1; rhs = false; } if (side < 0 || chain_in < 0 - || nodes[(size_t)side]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)side]->output_buffer.precision != run + || retyped((size_t)side) || nodes[(size_t)side]->output_buffer.total_size != total || plan->action[(size_t)side] == -2) { ok = false; break; } size_t slot = sides.size(); @@ -967,20 +998,26 @@ MetalFusePlan* cactus_metal_plan_build( if (slot == sides.size()) { if (sides.size() >= 3) { ok = false; break; } sides.push_back((size_t)side); - } - c2 |= (rhs ? 16 : 0) | ((int)slot << 5); - if (ci == 0) head_in = chain_in; - } else if (ci == 0) { - head_in = idxof(cn.input_ids[0]); + side_f32.push_back(run == Precision::FP32 ? 1 : 0); + } else if (side_f32[slot] != (run == Precision::FP32 ? 1 : 0)) { ok = false; break; } + c2 |= (rhs ? 16 : 0) | (run == Precision::FP32 ? 32 : 0) | ((int)slot << 6); } + if (k2 == 4) { + Precision from = ci == 0 ? run : nodes[(size_t)prev]->output_buffer.precision; + if (from == cn.output_buffer.precision) { ok = false; break; } + run = cn.output_buffer.precision; + } else if (cn.output_buffer.precision != run) { ok = false; break; } EwChainStep st{k2, c2, q0, q1}; float packed[4]; std::memcpy(packed, &st, sizeof(st)); blob.insert(blob.end(), packed, packed + 4); } - if (!ok || head_in < 0 - || nodes[(size_t)head_in]->output_buffer.precision != Precision::FP16 - || nodes[(size_t)head_in]->output_buffer.total_size != total) continue; + if (!ok || head_in < 0) continue; + Precision head_prec = nodes[(size_t)head_in]->output_buffer.precision; + uint32_t flags = (head_prec == Precision::FP32 ? 1u : 0u) + | (nodes[chain.back()]->output_buffer.precision == Precision::FP32 ? 2u : 0u); + for (size_t si = 0; si < sides.size(); ++si) + if (side_f32[si]) flags |= (4u << si); MetalCluster c; c.rule = 11; c.a0 = (size_t)head_in; @@ -990,6 +1027,7 @@ MetalFusePlan* cactus_metal_plan_build( c.u0 = (uint32_t)chain.size(); c.u1 = (uint32_t)sides.size(); c.b0 = plan->blobs.size(); + c.b1 = flags; c.b4 = chain.back(); plan->blobs.push_back(std::move(blob)); std::vector cover(chain.begin(), chain.end() - 1); @@ -1361,7 +1399,7 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, } const auto& blob = p->blobs[c.b0]; return cactus_metal_encode_elemwise_chain(out, in, blob.data(), c.u0, - sides[0], sides[1], sides[2], anchor.output_buffer.total_size); + sides[0], sides[1], sides[2], anchor.output_buffer.total_size, (uint32_t)c.b1); } default: return false; } diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 465e4e943..6a0ade2c2 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -148,11 +148,12 @@ bool cactus_metal_encode_groupnorm(void* out, const void* x, const void* w, cons bool cactus_metal_encode_bias_add_rows(void* y, const void* bias, uint32_t C, uint32_t total); bool cactus_metal_encode_elemwise_chain(void* out, const void* in, const float* steps, uint32_t nsteps, const void* side0, const void* side1, - const void* side2, size_t n); + const void* side2, size_t n, uint32_t flags); bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a); bool cactus_metal_encode_conv1d_dw(void* out, const void* x, const void* w, uint32_t C, uint32_t L, uint32_t Lout, uint32_t K, uint32_t stride); +bool cactus_metal_encode_transpose2d(void* out, const void* in, uint32_t batch, uint32_t R, uint32_t C); bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* oshape, const uint32_t* sstride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes); diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 1d3712317..fdf3eaf07 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -929,11 +929,16 @@ kernel void copy_bytes(device const uchar* in [[buffer(0)]], device uchar* out [ kernel void binary_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], device half* y [[buffer(2)]], constant uint& n [[buffer(3)]], constant int& op [[buffer(4)]], uint i [[thread_position_in_grid]]) { - if (i>=n) return; float av=(float)a[i], bv=(float)b[i], r; - switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; - case 5: r=(av!=bv)?1.0f:0.0f; break; default: r=av+bv; } - if (op==1) r=clamp(r,-65500.0f,65500.0f); - y[i]=(half)r; + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) { + float av=(float)a[i0+j], bv=(float)b[i0+j], r; + switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; + case 5: r=(av!=bv)?1.0f:0.0f; break; default: r=av+bv; } + if (op==1) r=clamp(r,-65500.0f,65500.0f); + y[i0+j]=(half)r; + } } kernel void rope_f16(device const half* x [[buffer(0)]], device half* out [[buffer(1)]], @@ -977,21 +982,28 @@ static inline float scalar_apply(float v, float p, int op) { kernel void scalar_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], constant float& p [[buffer(4)]], uint i [[thread_position_in_grid]]) { - if (i>=n) return; - y[i]=(half)scalar_apply((float)in[i], p, op); + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) y[i0+j]=(half)scalar_apply((float)in[i0+j], p, op); } kernel void unary_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], uint i [[thread_position_in_grid]]) { - if (i>=n) return; float x=(float)in[i], r; - if (op==0) r=gelu_tanh(x); - else if (op==1) r=precise::tanh(x); - else if (op==2) r=x/(1.0f+precise::exp(-x)); - else if (op==4) r=0.5f*x*(1.0f+erf_approx(x*0.70710678f)); - else if (op==5) r=1.0f/(1.0f+precise::exp(-x)); - else r=max(x,0.0f); - y[i]=(half)r; + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) { + float x=(float)in[i0+j], r; + if (op==0) r=gelu_tanh(x); + else if (op==1) r=precise::tanh(x); + else if (op==2) r=x/(1.0f+precise::exp(-x)); + else if (op==4) r=0.5f*x*(1.0f+erf_approx(x*0.70710678f)); + else if (op==5) r=1.0f/(1.0f+precise::exp(-x)); + else r=max(x,0.0f); + y[i0+j]=(half)r; + } } kernel void swiglu_f16(device const half* gate [[buffer(0)]], device const half* up [[buffer(1)]], @@ -1004,6 +1016,38 @@ kernel void swiglu_f16(device const half* gate [[buffer(0)]], device const half* y[i]=(half)((float)g2*(float)up[i]); } +kernel void rms_norm_simd_f16(device const half* in [[buffer(0)]], device const half* w [[buffer(1)]], + device half* y [[buffer(2)]], constant uint& dim [[buffer(3)]], + constant float& eps [[buffer(4)]], constant uint& rows [[buffer(5)]], + uint tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + uint row = tg * 4 + sgid; + if (row >= rows) return; + device const half* x = in + (size_t)row * dim; + device half* o = y + (size_t)row * dim; + float partial = 0; + for (uint i = lane * 4; i + 3 < dim; i += 128) { + half4 v4 = *(device const half4*)(x + i); + float4 f = float4(v4); + partial += f.x*f.x + f.y*f.y + f.z*f.z + f.w*f.w; + } + for (uint i = (dim & ~3u) + lane; i < dim; i += 32) { float v = (float)x[i]; partial += v*v; } + partial += simd_shuffle_xor(partial, 16); + partial += simd_shuffle_xor(partial, 8); + partial += simd_shuffle_xor(partial, 4); + partial += simd_shuffle_xor(partial, 2); + partial += simd_shuffle_xor(partial, 1); + float inv = 1.0f / sqrt(partial / (float)dim + eps); + for (uint i = lane * 4; i + 3 < dim; i += 128) { + half4 v4 = *(device const half4*)(x + i); + half4 w4 = *(device const half4*)(w + i); + float4 f = float4(v4) * inv * float4(w4); + *(device half4*)(o + i) = half4(f); + } + for (uint i = (dim & ~3u) + lane; i < dim; i += 32) o[i] = (half)((float)x[i] * inv * (float)w[i]); +} + kernel void rms_norm_f16(device const half* in [[buffer(0)]], device const half* w [[buffer(1)]], device half* y [[buffer(2)]], constant uint& dim [[buffer(3)]], constant float& eps [[buffer(4)]], @@ -1090,6 +1134,27 @@ kernel void strided_copy_f16(device const half* in [[buffer(0)]], device half* o out[i]=in[src]; } +kernel void transpose2d_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& R [[buffer(2)]], constant uint& C [[buffer(3)]], + uint3 tgid [[threadgroup_position_in_grid]], + uint3 tp3 [[thread_position_in_threadgroup]]) { + uint2 tp = tp3.xy; + threadgroup half tile[32][33]; + uint b = tgid.z; + uint c0 = tgid.x * 32, r0 = tgid.y * 32; + device const half* src = in + (size_t)b * R * C; + device half* dst = y + (size_t)b * R * C; + for (uint dr = tp.y; dr < 32; dr += 8) { + uint r = r0 + dr, c = c0 + tp.x; + tile[dr][tp.x] = (r < R && c < C) ? src[(size_t)r * C + c] : (half)0; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint dr = tp.y; dr < 32; dr += 8) { + uint c = c0 + dr, r = r0 + tp.x; + if (c < C && r < R) dst[(size_t)c * R + r] = tile[tp.x][dr]; + } +} + kernel void strided_copy_rows_f16(device const half* in [[buffer(0)]], device half* out [[buffer(1)]], constant uint* oshape [[buffer(2)]], constant uint* sstride [[buffer(3)]], constant uint& ndim [[buffer(4)]], constant uint& rows [[buffer(5)]], constant uint& base [[buffer(6)]], @@ -1198,22 +1263,42 @@ kernel void strided_scatter_f16(device const half* in [[buffer(0)]], device half out[dst]=in[i]; } +kernel void strided_scatter_rows_f16(device const half* in [[buffer(0)]], device half* out [[buffer(1)]], + constant uint* ishape [[buffer(2)]], constant uint* ostride [[buffer(3)]], + constant uint& ndim [[buffer(4)]], constant uint& rows [[buffer(5)]], constant uint& base [[buffer(6)]], + constant uint& inner4 [[buffer(7)]], + uint2 gid [[thread_position_in_grid]]) { + uint v = gid.x, row = gid.y; + if (v >= inner4 || row >= rows) return; + uint rem = row, dst = base; + for (int d = int(ndim) - 1; d >= 0; --d) { uint c = rem % ishape[d]; rem /= ishape[d]; dst += c * ostride[d]; } + device const half4* s4 = (device const half4*)(in + (size_t)row * inner4 * 4); + device half4* d4 = (device half4*)(out + dst); + d4[v] = s4[v]; +} + kernel void binary_f32(device const float* a [[buffer(0)]], device const float* b [[buffer(1)]], device float* y [[buffer(2)]], constant uint& n [[buffer(3)]], constant int& op [[buffer(4)]], uint i [[thread_position_in_grid]]) { - if (i>=n) return; float av=a[i], bv=b[i], r; - switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; case 5: r=(av!=bv)?1.0f:0.0f; break; default: r=av+bv; } - if (op==1) r=clamp(r,-65500.0f,65500.0f); - y[i]=r; + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) { + float av=a[i0+j], bv=b[i0+j], r; + switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; + case 5: r=(av!=bv)?1.0f:0.0f; break; default: r=av+bv; } + if (op==1) r=clamp(r,-65500.0f,65500.0f); + y[i0+j]=r; + } } - kernel void scalar_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], constant float& p [[buffer(4)]], uint i [[thread_position_in_grid]]) { - if (i>=n) return; - y[i]=scalar_apply(in[i], p, op); + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) y[i0+j]=scalar_apply(in[i0+j], p, op); } - kernel void unary_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], uint i [[thread_position_in_grid]]) { @@ -1230,15 +1315,20 @@ kernel void unary_f32(device const float* in [[buffer(0)]], device float* y [[bu kernel void clamp_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], constant uint& n [[buffer(2)]], constant float& lo [[buffer(3)]], constant float& hi [[buffer(4)]], uint i [[thread_position_in_grid]]) { - if (i= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) y[i0+j]=(half)clamp((float)in[i0+j], lo, hi); } kernel void clamp_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], constant uint& n [[buffer(2)]], constant float& lo [[buffer(3)]], constant float& hi [[buffer(4)]], uint i [[thread_position_in_grid]]) { - if (i= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) y[i0+j]=clamp(in[i0+j], lo, hi); } - kernel void glu_f16(device const half* x [[buffer(0)]], device half* y [[buffer(1)]], constant uint& split [[buffer(2)]], constant uint& inner [[buffer(3)]], constant uint& n [[buffer(4)]], uint i [[thread_position_in_grid]]) { @@ -1523,6 +1613,173 @@ kernel void attn_f16_d64(device const half* q [[buffer(0)]], device const half* } } +kernel void attn_flash_f16(device const half* q [[buffer(0)]], device const half* k [[buffer(1)]], + device const half* v [[buffer(2)]], device half* y [[buffer(3)]], + device const half* mask [[buffer(4)]], + constant uint& T [[buffer(5)]], constant uint& S [[buffer(6)]], + constant uint& H [[buffer(7)]], constant float& scale [[buffer(8)]], + constant uint& mask_mode [[buffer(9)]], + uint2 tgid [[threadgroup_position_in_grid]], + uint tid [[thread_index_in_threadgroup]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + constexpr uint BQ = 64, BK = 32, D = 64, NT = 256; + threadgroup half Ks[BK * D]; + threadgroup half Vs[BK * D]; + threadgroup float Ss[8][8 * BK]; + threadgroup half Ps[8][8 * BK]; + threadgroup half Dg[8][64]; + threadgroup float Ms[8][8]; + threadgroup float Ls[8][8]; + threadgroup uint Rf[8]; + + uint qb = tgid.x, h = tgid.y; + uint q0 = qb * BQ; + uint row0 = q0 + sgid * 8; + bool interior = row0 + 8 <= T; + + for (uint x = tid; x < 8 * 64; x += NT) Dg[x / 64][x % 64] = 0; + if (tid < 64) { Ms[tid / 8][tid % 8] = -INFINITY; Ls[tid / 8][tid % 8] = 0; } + + simdgroup_half8x8 qf[8]; + if (interior) { + for (uint d = 0; d < 8; ++d) + simdgroup_load(qf[d], q + ((size_t)row0 * H + h) * D + d * 8, (size_t)H * D); + } else { + threadgroup half* Qe = Ks; + for (uint x = tid; x < BQ * D; x += NT) { + uint r = x / D, d = x % D; + uint t = q0 + r; + Qe[x] = (t < T) ? q[((size_t)t * H + h) * D + d] : (half)0; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint d = 0; d < 8; ++d) + simdgroup_load(qf[d], Qe + (size_t)sgid * 8 * D + d * 8, D); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + simdgroup_half8x8 of[8]; + for (uint d = 0; d < 8; ++d) of[d] = make_filled_simdgroup_matrix(0.0h); + + for (uint kb = 0; kb < S; kb += BK) { + for (uint x = tid; x < BK * D / 4; x += NT) { + uint r = (x * 4) / D, d = (x * 4) % D; + uint sidx = kb + r; + half4 kv4 = half4(0.0h), vv4 = half4(0.0h); + if (sidx < S) { + kv4 = *(device const half4*)(k + ((size_t)sidx * H + h) * D + d); + vv4 = *(device const half4*)(v + ((size_t)sidx * H + h) * D + d); + } + *(threadgroup half4*)(Ks + r * D + d) = kv4; + *(threadgroup half4*)(Vs + r * D + d) = vv4; + } + if (lane == 0) Rf[sgid] = 0; + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint j = 0; j < 4; ++j) { + simdgroup_float8x8 sf = make_filled_simdgroup_matrix(0.0f); + for (uint d = 0; d < 8; ++d) { + simdgroup_half8x8 kt; + simdgroup_load(kt, Ks + (size_t)j * 8 * D + d * 8, D, 0, true); + simdgroup_multiply_accumulate(sf, qf[d], kt, sf); + } + simdgroup_store(sf, Ss[sgid] + j * 8, BK); + } + simdgroup_barrier(mem_flags::mem_threadgroup); + + uint r = lane / 4, c0 = (lane % 4) * 8; + uint t = row0 + r; + float sv[8]; + float mloc = -INFINITY; + for (uint c = 0; c < 8; ++c) { + uint sidx = kb + c0 + c; + float val = Ss[sgid][r * BK + c0 + c] * scale; + bool dead = sidx >= S || t >= T; + if (!dead && mask_mode == 2u && mask[(size_t)t * S + sidx] == (half)0.0h) dead = true; + sv[c] = dead ? -INFINITY : val; + mloc = max(mloc, sv[c]); + } + mloc = max(mloc, simd_shuffle_xor(mloc, 1)); + mloc = max(mloc, simd_shuffle_xor(mloc, 2)); + float mold = Ms[sgid][r]; + float mnew = max(mold, mloc); + float sloc = 0; + for (uint c = 0; c < 8; ++c) { + float pv = (mnew == -INFINITY || sv[c] == -INFINITY) ? 0.0f : fast::exp(sv[c] - mnew); + Ps[sgid][r * BK + c0 + c] = (half)pv; + sloc += pv; + } + sloc += simd_shuffle_xor(sloc, 1); + sloc += simd_shuffle_xor(sloc, 2); + float alpha = (mold == -INFINITY) ? 0.0f : fast::exp(mold - mnew); + bool first = mold == -INFINITY; + if ((lane % 4) == 0) { + Ms[sgid][r] = mnew; + Ls[sgid][r] = Ls[sgid][r] * alpha + sloc; + if (mnew != mold && !first) Rf[sgid] = 1; + Dg[sgid][r * 8 + r] = (half)alpha; + } + simdgroup_barrier(mem_flags::mem_threadgroup); + + bool rescale = Rf[sgid] != 0 || kb == 0; + simdgroup_half8x8 pf[4]; + for (uint j = 0; j < 4; ++j) + simdgroup_load(pf[j], Ps[sgid] + j * 8, BK); + if (rescale) { + simdgroup_half8x8 dgf; + simdgroup_load(dgf, Dg[sgid], 8); + for (uint d = 0; d < 8; ++d) { + simdgroup_half8x8 acc; + simdgroup_multiply(acc, dgf, of[d]); + for (uint j = 0; j < 4; ++j) { + simdgroup_half8x8 vf; + simdgroup_load(vf, Vs + (size_t)j * 8 * D + d * 8, D); + simdgroup_multiply_accumulate(acc, pf[j], vf, acc); + } + of[d] = acc; + } + } else { + for (uint d = 0; d < 8; ++d) { + for (uint j = 0; j < 4; ++j) { + simdgroup_half8x8 vf; + simdgroup_load(vf, Vs + (size_t)j * 8 * D + d * 8, D); + simdgroup_multiply_accumulate(of[d], pf[j], vf, of[d]); + } + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + { + uint r = lane / 4; + if ((lane % 4) == 0) { + float l = Ls[sgid][r]; + Dg[sgid][r * 8 + r] = (half)(l > 0.0f ? 1.0f / l : 0.0f); + } + simdgroup_barrier(mem_flags::mem_threadgroup); + simdgroup_half8x8 dgf; + simdgroup_load(dgf, Dg[sgid], 8); + if (interior) { + for (uint d = 0; d < 8; ++d) { + simdgroup_multiply(of[d], dgf, of[d]); + simdgroup_store(of[d], y + ((size_t)row0 * H + h) * D + d * 8, (size_t)H * D); + } + } else { + threadgroup half* Oe = Ks; + for (uint d = 0; d < 8; ++d) { + simdgroup_multiply(of[d], dgf, of[d]); + simdgroup_store(of[d], Oe + (size_t)sgid * 8 * D + d * 8, D); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint x = tid; x < BQ * D; x += NT) { + uint r2 = x / D, d = x % D; + uint t = q0 + r2; + if (t < T) y[((size_t)t * H + h) * D + d] = Oe[x]; + } + } + } +} + kernel void attn_maskfill_f16(device const half* mask [[buffer(0)]], device half* out [[buffer(1)]], constant uint& ts [[buffer(2)]], uint2 gid [[thread_position_in_grid]]) { @@ -1796,25 +2053,31 @@ kernel void conv2d_f16(device const half* x [[buffer(0)]], device const half* w constant uint& pad [[buffer(12)]], constant uint& dw [[buffer(13)]], constant uint& has_bias [[buffer(14)]], uint3 gid [[thread_position_in_grid]]) { - uint wo = gid.x, ho = gid.y, nco = gid.z; - if (wo >= Wo || ho >= Ho) return; + uint wo0 = gid.x * 4, ho = gid.y, nco = gid.z; + if (wo0 >= Wo || ho >= Ho) return; uint n = nco / Cout, co = nco % Cout; - float acc = has_bias ? (float)bias[co] : 0.0f; + float acc[4]; + float bv = has_bias ? (float)bias[co] : 0.0f; + for (uint q = 0; q < 4; ++q) acc[q] = bv; uint ci0 = dw ? co : 0, ci1 = dw ? co + 1 : Cin; for (uint ci = ci0; ci < ci1; ++ci) { for (uint kh = 0; kh < K; ++kh) { long h = (long)ho * stride - (long)pad + kh; if (h < 0 || h >= (long)H) continue; + device const half* xr = x + (((size_t)n*Cin + ci)*H + h)*W; for (uint kw = 0; kw < K; ++kw) { - long ww = (long)wo * stride - (long)pad + kw; - if (ww < 0 || ww >= (long)W) continue; float wv = dw ? (float)w[((size_t)co*K + kh)*K + kw] : (float)w[(((size_t)co*Cin + ci)*K + kh)*K + kw]; - acc = fma((float)x[(((size_t)n*Cin + ci)*H + h)*W + ww], wv, acc); + for (uint q = 0; q < 4; ++q) { + long ww = (long)(wo0 + q) * stride - (long)pad + kw; + if (ww >= 0 && ww < (long)W && wo0 + q < Wo) + acc[q] = fma((float)xr[ww], wv, acc[q]); + } } } } - y[(((size_t)n*Cout + co)*Ho + ho)*Wo + wo] = (half)acc; + for (uint q = 0; q < 4 && wo0 + q < Wo; ++q) + y[(((size_t)n*Cout + co)*Ho + ho)*Wo + wo0 + q] = (half)acc[q]; } kernel void batchnorm_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], @@ -1863,41 +2126,74 @@ kernel void bias_add_rows_f16(device half* y [[buffer(0)]], device const half* b struct EwStep { int kind; int code; float p0; float p1; }; -kernel void elemwise_chain_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], +kernel void elemwise_chain_f16(device const uchar* in [[buffer(0)]], device uchar* y [[buffer(1)]], constant EwStep* steps [[buffer(2)]], constant uint& nsteps [[buffer(3)]], - device const half* s0 [[buffer(4)]], device const half* s1 [[buffer(5)]], - device const half* s2 [[buffer(6)]], constant uint& n [[buffer(7)]], - uint i [[thread_position_in_grid]]) { - if (i >= n) return; - float x = (float)in[i]; + device const uchar* s0 [[buffer(4)]], device const uchar* s1 [[buffer(5)]], + device const uchar* s2 [[buffer(6)]], constant uint& n [[buffer(7)]], + constant uint& flags [[buffer(8)]], + uint gid [[thread_position_in_grid]]) { + uint i0 = gid * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + float xv[4]; + bool in32 = (flags & 1u) != 0; + for (uint q = 0; q < cnt; ++q) + xv[q] = in32 ? ((device const float*)in)[i0+q] : (float)((device const half*)in)[i0+q]; + bool prec32 = in32; for (uint s = 0; s < nsteps; ++s) { EwStep st = steps[s]; - if (st.kind == 0) { - if (st.code == 0) x = gelu_tanh(x); - else if (st.code == 1) x = precise::tanh(x); - else if (st.code == 2) x = x/(1.0f+precise::exp(-x)); - else if (st.code == 4) x = 0.5f*x*(1.0f+erf_approx(x*0.70710678f)); - else if (st.code == 5) x = 1.0f/(1.0f+precise::exp(-x)); - else x = max(x, 0.0f); - } else if (st.kind == 1) { - x = scalar_apply(x, st.p0, st.code); - } else if (st.kind == 2) { - uint slot = st.code >> 5; + if (st.kind == 4) { prec32 = st.code != 0; continue; } + if (st.kind == 2) { + uint slot = (st.code >> 6) & 3u; int op = st.code & 15; bool rhs = (st.code & 16) != 0; - float o = (float)(slot == 0 ? s0[i] : slot == 1 ? s1[i] : s2[i]); - float a = rhs ? o : x, b = rhs ? x : o; - float r; - switch (op) { case 2: r=a-b; break; case 3: r=a*b; break; case 4: r=a/b; break; - case 5: r=(a!=b)?1.0f:0.0f; break; default: r=a+b; } - if (op == 1) r = clamp(r, -65500.0f, 65500.0f); - x = r; + bool sf32 = (st.code & 32) != 0; + device const uchar* sp = slot == 0 ? s0 : slot == 1 ? s1 : s2; + for (uint q = 0; q < cnt; ++q) { + float o = sf32 ? ((device const float*)sp)[i0+q] : (float)((device const half*)sp)[i0+q]; + float a = rhs ? o : xv[q], b = rhs ? xv[q] : o; + float r; + switch (op) { case 2: r=a-b; break; case 3: r=a*b; break; case 4: r=a/b; break; + case 5: r=(a!=b)?1.0f:0.0f; break; default: r=a+b; } + if (op == 1) r = clamp(r, -65500.0f, 65500.0f); + xv[q] = r; + } + } else if (st.kind == 0) { + for (uint q = 0; q < cnt; ++q) { + float x = xv[q]; + if (st.code == 0) x = gelu_tanh(x); + else if (st.code == 1) x = precise::tanh(x); + else if (st.code == 2) x = x/(1.0f+precise::exp(-x)); + else if (st.code == 4) x = 0.5f*x*(1.0f+erf_approx(x*0.70710678f)); + else if (st.code == 5) x = 1.0f/(1.0f+precise::exp(-x)); + else x = max(x, 0.0f); + xv[q] = x; + } + } else if (st.kind == 1) { + for (uint q = 0; q < cnt; ++q) xv[q] = scalar_apply(xv[q], st.p0, st.code); } else { - x = clamp(x, st.p0, st.p1); + for (uint q = 0; q < cnt; ++q) xv[q] = clamp(xv[q], st.p0, st.p1); } - x = (float)(half)x; + if (!prec32) for (uint q = 0; q < cnt; ++q) xv[q] = (float)(half)xv[q]; } - y[i] = (half)x; + if ((flags & 2u) != 0) { for (uint q = 0; q < cnt; ++q) ((device float*)y)[i0+q] = xv[q]; } + else { for (uint q = 0; q < cnt; ++q) ((device half*)y)[i0+q] = (half)xv[q]; } +} + +kernel void bcast_binary_rows_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], + device half* out [[buffer(2)]], constant uint* oshape [[buffer(3)]], + constant uint* astride [[buffer(4)]], constant uint* bstride [[buffer(5)]], + constant uint& ndim [[buffer(6)]], constant uint& rows [[buffer(7)]], constant int& op [[buffer(8)]], + constant uint& inner [[buffer(9)]], constant uint& ainner [[buffer(10)]], constant uint& binner [[buffer(11)]], + uint2 gid [[thread_position_in_grid]]) { + uint v = gid.x, row = gid.y; + if (v >= inner || row >= rows) return; + uint rem = row, ai = 0, bi = 0; + for (int d = int(ndim) - 1; d >= 0; --d) { uint c = rem % oshape[d]; rem /= oshape[d]; ai += c * astride[d]; bi += c * bstride[d]; } + float av = (float)a[ai + v * ainner], bv = (float)b[bi + v * binner], r; + switch(op){ case 2:r=av-bv;break; case 3:r=av*bv;break; case 4:r=av/bv;break; default:r=av+bv; } + if (op==1) r=clamp(r,-65500.0f,65500.0f); + out[(size_t)row * inner + v] = (half)r; } kernel void bcast_binary_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index a6c64d593..047339784 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -55,6 +55,8 @@ id psoConcat2=nil, psoGatherF32Idx=nil, psoRopeFull=nil, psoMaxpool1d=nil; id psoBilinear=nil, psoConv1dGen=nil, psoConv1dNlcDw=nil, psoConv2d=nil; id psoBatchnorm=nil, psoGroupnorm=nil, psoBiasAddRows=nil, psoEwChain=nil; + id psoAttnFlash=nil, psoRmsSimd=nil, psoScatterRows=nil; + id psoTranspose2d=nil, psoBcastRows=nil; id dummy=nil; bool ok = false; @@ -110,6 +112,9 @@ psoConv1dNlcDw=pso("conv1d_nlc_dw_f16"); psoConv2d=pso("conv2d_f16"); psoBatchnorm=pso("batchnorm_f16"); psoGroupnorm=pso("groupnorm_f16"); psoBiasAddRows=pso("bias_add_rows_f16"); psoEwChain=pso("elemwise_chain_f16"); + psoAttnFlash=pso("attn_flash_f16"); psoRmsSimd=pso("rms_norm_simd_f16"); + psoScatterRows=pso("strided_scatter_rows_f16"); psoTranspose2d=pso("transpose2d_f16"); + psoBcastRows=pso("bcast_binary_rows_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; @@ -435,7 +440,7 @@ bool cactus_metal_encode_binary(int op, void* out, const void* a, const void* b, [g_enc setComputePipelineState:ctx().psoBinary]; setBufAt(a, n*2, 0); setBufAt(b, n*2, 1); setBufAt(out, n*2, 2); [g_enc setBytes:&nn length:4 atIndex:3]; [g_enc setBytes:&o length:4 atIndex:4]; - [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + [g_enc dispatchThreads:MTLSizeMake((nn+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_scalar(int op, void* out, const void* in, size_t n, float param) { @@ -445,7 +450,7 @@ bool cactus_metal_encode_scalar(int op, void* out, const void* in, size_t n, flo [g_enc setComputePipelineState:ctx().psoScalar]; setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; [g_enc setBytes:&p length:4 atIndex:4]; - [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + [g_enc dispatchThreads:MTLSizeMake((nn+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_unary(int op, void* out, const void* in, size_t n) { @@ -455,7 +460,7 @@ bool cactus_metal_encode_unary(int op, void* out, const void* in, size_t n) { [g_enc setComputePipelineState:ctx().psoUnary]; setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; - [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + [g_enc dispatchThreads:MTLSizeMake((nn+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, size_t n, float scale) { @@ -485,6 +490,15 @@ bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, if (!ctx().ok) return false; ensureEncoder(); uint32_t d=(uint32_t)dim; float e=eps; + if (rows > 1 && ctx().psoRmsSimd) { + uint32_t r=(uint32_t)rows; + [g_enc setComputePipelineState:ctx().psoRmsSimd]; + setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(out, rows*dim*2, 2); + [g_enc setBytes:&d length:4 atIndex:3]; [g_enc setBytes:&e length:4 atIndex:4]; + [g_enc setBytes:&r length:4 atIndex:5]; + [g_enc dispatchThreadgroups:MTLSizeMake((rows+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + return true; + } [g_enc setComputePipelineState:ctx().psoRms]; setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(out, rows*dim*2, 2); [g_enc setBytes:&d length:4 atIndex:3]; [g_enc setBytes:&e length:4 atIndex:4]; @@ -1250,7 +1264,7 @@ bool cactus_metal_encode_binary_f32(int op, void* out, const void* a, const void [g_enc setComputePipelineState:ctx().psoBinF32]; setBufAt(a, n*4, 0); setBufAt(b, n*4, 1); setBufAt(out, n*4, 2); [g_enc setBytes:&nn length:4 atIndex:3]; [g_enc setBytes:&o length:4 atIndex:4]; - [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + [g_enc dispatchThreads:MTLSizeMake((n+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_scalar_f32(int op, void* out, const void* in, size_t n, float p) { @@ -1260,7 +1274,7 @@ bool cactus_metal_encode_scalar_f32(int op, void* out, const void* in, size_t n, [g_enc setComputePipelineState:ctx().psoScaF32]; setBufAt(in, n*4, 0); setBufAt(out, n*4, 1); [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; [g_enc setBytes:&p length:4 atIndex:4]; - [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + [g_enc dispatchThreads:MTLSizeMake((n+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_unary_f32(int op, void* out, const void* in, size_t n) { @@ -1281,7 +1295,8 @@ bool cactus_metal_encode_clamp(void* out, const void* in, size_t n, float lo, fl [g_enc setComputePipelineState:pso]; setBufAt(in, n*es, 0); setBufAt(out, n*es, 1); [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&lo length:4 atIndex:3]; [g_enc setBytes:&hi length:4 atIndex:4]; - [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + size_t grid = (n + 3) / 4; + [g_enc dispatchThreads:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_glu(void* out, const void* in, size_t split, size_t inner, size_t n_out) { @@ -1442,6 +1457,21 @@ bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, if (D > 128 || DV > 128 || (D % 4) != 0 || (DV % 4) != 0 || HKV == 0 || HQ % HKV != 0) return false; static const bool mps_attn_ok = MPSSupportsMTLDevice(ctx().dev); bool mask_ok = mask_mode == 0 || (mask_mode == 2 && mask && ctx().psoMaskFill); + if (ctx().psoAttnFlash && B == 1 && HQ == HKV && !causal && window == 0 + && logit_cap == 0.0f && D == 64 && DV == 64 && S >= 256 + && (mask_mode == 0 || (mask_mode == 2 && mask))) { + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoAttnFlash]; + setBufAt(q, (size_t)T*HQ*D*2, 0); setBufAt(k, (size_t)S*HKV*D*2, 1); + setBufAt(v, (size_t)S*HKV*DV*2, 2); setBufAt(out, (size_t)T*HQ*DV*2, 3); + if (mask_mode == 2 && mask) setBufAt(mask, (size_t)T*S*2, 4); + else [g_enc setBuffer:ctx().dummy offset:0 atIndex:4]; + [g_enc setBytes:&T length:4 atIndex:5]; [g_enc setBytes:&S length:4 atIndex:6]; + [g_enc setBytes:&HQ length:4 atIndex:7]; [g_enc setBytes:&scale length:4 atIndex:8]; + [g_enc setBytes:&mask_mode length:4 atIndex:9]; + [g_enc dispatchThreadgroups:MTLSizeMake((T + 63) / 64, HQ, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)]; + return true; + } if (mps_attn_ok && B == 1 && HQ == HKV && !causal && mask_ok && window == 0 && logit_cap == 0.0f && S >= 512 && (S % 8) == 0 && (D % 8) == 0 && (DV % 8) == 0 && ctx().psoSoftmaxR) { @@ -1664,7 +1694,8 @@ bool cactus_metal_encode_conv2d(void* out, const void* x, const void* w, const v [g_enc setBytes:&K length:4 atIndex:10]; [g_enc setBytes:&stride length:4 atIndex:11]; [g_enc setBytes:&pad length:4 atIndex:12]; [g_enc setBytes:&dwf length:4 atIndex:13]; [g_enc setBytes:&hb length:4 atIndex:14]; - [g_enc dispatchThreads:MTLSizeMake(Wo, Ho, (size_t)N*Cout) threadsPerThreadgroup:MTLSizeMake(std::min(Wo, 32u), std::min(Ho, 8u), 1)]; + uint32_t wq = (Wo + 3) / 4; + [g_enc dispatchThreads:MTLSizeMake(wq, Ho, (size_t)N*Cout) threadsPerThreadgroup:MTLSizeMake(std::min(wq, 32u), std::min(Ho, 8u), 1)]; return true; } bool cactus_metal_encode_batchnorm(void* out, const void* x, const void* w, const void* b, @@ -1705,19 +1736,21 @@ bool cactus_metal_encode_bias_add_rows(void* y, const void* bias, uint32_t C, ui } bool cactus_metal_encode_elemwise_chain(void* out, const void* in, const float* steps, uint32_t nsteps, const void* side0, const void* side1, - const void* side2, size_t n) { - if (!ctx().ok || !ctx().psoEwChain || nsteps == 0 || nsteps > 8) return false; + const void* side2, size_t n, uint32_t flags) { + if (!ctx().ok || !ctx().psoEwChain || nsteps == 0 || nsteps > 12) return false; ensureEncoder(); uint32_t ns = nsteps, nn = (uint32_t)n; + size_t ein = (flags & 1u) ? 4 : 2, eout = (flags & 2u) ? 4 : 2; [g_enc setComputePipelineState:ctx().psoEwChain]; - setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); + setBufAt(in, n*ein, 0); setBufAt(out, n*eout, 1); [g_enc setBytes:steps length:nsteps*16 atIndex:2]; [g_enc setBytes:&ns length:4 atIndex:3]; - setBufAt(side0 ? side0 : in, n*2, 4); - setBufAt(side1 ? side1 : in, n*2, 5); - setBufAt(side2 ? side2 : in, n*2, 6); + setBufAt(side0 ? side0 : in, n*((flags & 4u) ? 4 : 2), 4); + setBufAt(side1 ? side1 : in, n*((flags & 8u) ? 4 : 2), 5); + setBufAt(side2 ? side2 : in, n*((flags & 16u) ? 4 : 2), 6); [g_enc setBytes:&nn length:4 atIndex:7]; - [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + [g_enc setBytes:&flags length:4 atIndex:8]; + [g_enc dispatchThreads:MTLSizeMake((nn+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, @@ -1744,6 +1777,15 @@ bool cactus_metal_encode_conv1d_dw(void* out, const void* x, const void* w, [g_enc dispatchThreads:MTLSizeMake(Lout, C, 1) threadsPerThreadgroup:MTLSizeMake(64,4,1)]; return true; } +bool cactus_metal_encode_transpose2d(void* out, const void* in, uint32_t batch, uint32_t R, uint32_t C) { + if (!ctx().ok || !ctx().psoTranspose2d) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoTranspose2d]; + setBufAt(in, (size_t)batch*R*C*2, 0); setBufAt(out, (size_t)batch*R*C*2, 1); + [g_enc setBytes:&R length:4 atIndex:2]; [g_enc setBytes:&C length:4 atIndex:3]; + [g_enc dispatchThreadgroups:MTLSizeMake((C+31)/32, (R+31)/32, batch) threadsPerThreadgroup:MTLSizeMake(32,8,1)]; + return true; +} bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* oshape, const uint32_t* sstride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes) { if (!ctx().ok || ndim == 0 || ndim > 8) return false; @@ -1774,6 +1816,21 @@ bool cactus_metal_encode_bcast_binary(int op, void* out, const void* a, const vo if (!ctx().ok || ndim == 0 || ndim > 8) return false; ensureEncoder(); int o = op; + if (ndim >= 2 && oshape[ndim-1] > 1 && ctx().psoBcastRows) { + uint32_t inner = oshape[ndim-1]; + uint32_t ainner = astride[ndim-1], binner = bstride[ndim-1]; + uint32_t nd2 = ndim - 1; + uint32_t rows = total / inner; + [g_enc setComputePipelineState:ctx().psoBcastRows]; + setBufAt(a, a_bytes, 0); setBufAt(b, b_bytes, 1); setBufAt(out, out_bytes, 2); + [g_enc setBytes:oshape length:nd2*4 atIndex:3]; [g_enc setBytes:astride length:nd2*4 atIndex:4]; + [g_enc setBytes:bstride length:nd2*4 atIndex:5]; + [g_enc setBytes:&nd2 length:4 atIndex:6]; [g_enc setBytes:&rows length:4 atIndex:7]; + [g_enc setBytes:&o length:4 atIndex:8]; [g_enc setBytes:&inner length:4 atIndex:9]; + [g_enc setBytes:&ainner length:4 atIndex:10]; [g_enc setBytes:&binner length:4 atIndex:11]; + [g_enc dispatchThreads:MTLSizeMake(inner, rows, 1) threadsPerThreadgroup:MTLSizeMake(std::min(inner, 64u), 4, 1)]; + return true; + } [g_enc setComputePipelineState:ctx().psoBcast]; setBufAt(a, a_bytes, 0); setBufAt(b, b_bytes, 1); setBufAt(out, out_bytes, 2); [g_enc setBytes:oshape length:ndim*4 atIndex:3]; [g_enc setBytes:astride length:ndim*4 atIndex:4]; [g_enc setBytes:bstride length:ndim*4 atIndex:5]; @@ -1785,6 +1842,19 @@ bool cactus_metal_encode_strided_scatter(void* out, const void* in, const uint32 const uint32_t* ostride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes) { if (!ctx().ok || ndim == 0 || ndim > 8) return false; ensureEncoder(); + if (ndim >= 2 && ostride[ndim-1] == 1 && (ishape[ndim-1] % 4) == 0 && ctx().psoScatterRows) { + uint32_t inner = ishape[ndim-1]; + uint32_t inner4 = inner / 4; + uint32_t nd2 = ndim - 1; + uint32_t rows = total / inner; + [g_enc setComputePipelineState:ctx().psoScatterRows]; + setBufAt(in, in_bytes, 0); setBufAt(out, out_bytes, 1); + [g_enc setBytes:ishape length:nd2*4 atIndex:2]; [g_enc setBytes:ostride length:nd2*4 atIndex:3]; + [g_enc setBytes:&nd2 length:4 atIndex:4]; [g_enc setBytes:&rows length:4 atIndex:5]; + [g_enc setBytes:&base length:4 atIndex:6]; [g_enc setBytes:&inner4 length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake(inner4, rows, 1) threadsPerThreadgroup:MTLSizeMake(std::min(inner4, 64u), 4, 1)]; + return true; + } [g_enc setComputePipelineState:ctx().psoScatter]; setBufAt(in, in_bytes, 0); setBufAt(out, out_bytes, 1); [g_enc setBytes:ishape length:ndim*4 atIndex:2]; [g_enc setBytes:ostride length:ndim*4 atIndex:3]; diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 71adf3516..f3fef8487 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -87,9 +87,10 @@ bool cactus_metal_encode_conv2d(void*, const void*, const void*, const void*, ui bool cactus_metal_encode_batchnorm(void*, const void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, float) { return false; } bool cactus_metal_encode_groupnorm(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, float) { return false; } bool cactus_metal_encode_bias_add_rows(void*, const void*, uint32_t, uint32_t) { return false; } -bool cactus_metal_encode_elemwise_chain(void*, const void*, const float*, uint32_t, const void*, const void*, const void*, size_t) { return false; } +bool cactus_metal_encode_elemwise_chain(void*, const void*, const float*, uint32_t, const void*, const void*, const void*, size_t, uint32_t) { return false; } bool cactus_metal_encode_gemm_batch(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, int, int) { return false; } bool cactus_metal_encode_conv1d_dw(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_transpose2d(void*, const void*, uint32_t, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_strided_copy(void*, const void*, const uint32_t*, const uint32_t*, uint32_t, uint32_t, uint32_t, size_t, size_t) { return false; } bool cactus_metal_encode_bcast_binary(int, void*, const void*, const void*, const uint32_t*, From 55d8986def2c56147bc7b02e3b7bba783a039bde Mon Sep 17 00:00:00 2001 From: jakmro Date: Fri, 3 Jul 2026 00:02:15 +0200 Subject: [PATCH 22/36] Epilogue-style fusion: broadcast chain sides and fused residual-add rms_norm - elementwise chains accept broadcast side inputs (per-row and per-column) so post-matmul scale and bias passes fold into the surrounding chain; side buffer bindings are sized per side - rule 12 fuses ADD/ADD_CLIPPED feeding a multi-row RMS_NORM into one rms_norm_add_simd_f16 dispatch that writes both the residual sum (kept via the skip-but-preallocate action) and the normalized output, matching the unfused fp16 rounding exactly All graph, parity, and engine suites pass; towers audio 143ms vision 242ms, decode and prefill unchanged. Signed-off-by: jakmro --- cactus-graph/src/metal_plan.cpp | 68 ++++++++++++++++++++++- cactus-kernels/metal_backend.h | 6 +- cactus-kernels/src/cactus_kernels.metal | 39 ++++++++++++- cactus-kernels/src/metal_backend.mm | 31 +++++++++-- cactus-kernels/src/metal_backend_stub.cpp | 3 +- 5 files changed, 135 insertions(+), 12 deletions(-) diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index 0ffed9e2a..e9900d040 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -924,6 +924,44 @@ MetalFusePlan* cactus_metal_plan_build( }; auto prec_ok = [](Precision p) { return p == Precision::FP16 || p == Precision::FP32; }; auto retyped = [&](size_t idx) { return idx < retype.size() && retype[idx] != 0; }; + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd0 = *nodes[i]; + if (nd0.op_type != OpType::RMS_NORM || nd0.input_ids.size() < 2) continue; + if (nd0.output_buffer.precision != Precision::FP16 || retyped(i)) continue; + const auto& osh = nd0.output_buffer.shape; + if (osh.empty()) continue; + size_t dim = osh.back(); + size_t rows = nd0.output_buffer.total_size / dim; + if (rows < 2 || dim < 32) continue; + long ai = idxof(nd0.input_ids[0]); + long wi = idxof(nd0.input_ids[1]); + if (ai < 0 || wi < 0 || plan->action[(size_t)ai] != -1 || retyped((size_t)ai)) continue; + const GraphNode& an = *nodes[(size_t)ai]; + if ((an.op_type != OpType::ADD && an.op_type != OpType::ADD_CLIPPED) + || an.input_ids.size() != 2 + || an.output_buffer.precision != Precision::FP16 + || an.output_buffer.total_size != nd0.output_buffer.total_size + || pinned[(size_t)ai]) continue; + long x0 = idxof(an.input_ids[0]); + long x1 = idxof(an.input_ids[1]); + if (x0 < 0 || x1 < 0 + || nodes[(size_t)x0]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)x1]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)x0]->output_buffer.total_size != an.output_buffer.total_size + || nodes[(size_t)x1]->output_buffer.total_size != an.output_buffer.total_size) continue; + long ww = deep_f16_source(wi); + if (ww < 0 || nodes[(size_t)ww]->output_buffer.total_size != dim) continue; + MetalCluster c; + c.rule = 12; + c.a0 = (size_t)x0; c.a1 = (size_t)x1; c.a2 = (size_t)ai; c.a3 = (size_t)ww; + c.u0 = (uint32_t)rows; c.u1 = (uint32_t)dim; + c.f0 = nd0.params.epsilon; + c.b0 = an.op_type == OpType::ADD_CLIPPED ? 1 : 0; + c.b4 = i; + std::vector cover; + if (add_cluster(c, i, cover)) plan->action[(size_t)ai] = -3; + } for (size_t i = 0; i < n; ++i) { if (plan->action[i] != -1) continue; int kind, code; float p0, p1; @@ -990,8 +1028,17 @@ MetalFusePlan* cactus_metal_plan_build( if (side < 0 || chain_in < 0 || nodes[(size_t)side]->output_buffer.precision != run || retyped((size_t)side) - || nodes[(size_t)side]->output_buffer.total_size != total || plan->action[(size_t)side] == -2) { ok = false; break; } + size_t inner = nodes[i]->output_buffer.shape.empty() + ? 1 : nodes[i]->output_buffer.shape.back(); + size_t stotal = nodes[(size_t)side]->output_buffer.total_size; + int bmode; + if (stotal == total) bmode = 0; + else if (inner > 1 && stotal * inner == total + && !nodes[(size_t)side]->output_buffer.shape.empty() + && nodes[(size_t)side]->output_buffer.shape.back() == 1) bmode = 1; + else if (inner > 1 && stotal == inner) bmode = 2; + else { ok = false; break; } size_t slot = sides.size(); for (size_t si = 0; si < sides.size(); ++si) if (sides[si] == (size_t)side) slot = si; @@ -1000,7 +1047,7 @@ MetalFusePlan* cactus_metal_plan_build( sides.push_back((size_t)side); side_f32.push_back(run == Precision::FP32 ? 1 : 0); } else if (side_f32[slot] != (run == Precision::FP32 ? 1 : 0)) { ok = false; break; } - c2 |= (rhs ? 16 : 0) | (run == Precision::FP32 ? 32 : 0) | ((int)slot << 6); + c2 |= (rhs ? 16 : 0) | (run == Precision::FP32 ? 32 : 0) | ((int)slot << 6) | (bmode << 8); } if (k2 == 4) { Precision from = ci == 0 ? run : nodes[(size_t)prev]->output_buffer.precision; @@ -1392,14 +1439,29 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, void* out = anchor.output_buffer.get_data(); if (!in || !out) return false; const void* sides[3] = { nullptr, nullptr, nullptr }; + size_t side_elems[3] = { 0, 0, 0 }; size_t side_idx[3] = { c.a1, c.a2, c.a3 }; for (uint32_t si = 0; si < c.u1; ++si) { sides[si] = nodes[side_idx[si]]->output_buffer.get_data(); + side_elems[si] = nodes[side_idx[si]]->output_buffer.total_size; if (!sides[si]) return false; } const auto& blob = p->blobs[c.b0]; + size_t inner = anchor.output_buffer.shape.empty() ? 1 : anchor.output_buffer.shape.back(); return cactus_metal_encode_elemwise_chain(out, in, blob.data(), c.u0, - sides[0], sides[1], sides[2], anchor.output_buffer.total_size, (uint32_t)c.b1); + sides[0], sides[1], sides[2], side_elems, anchor.output_buffer.total_size, + (uint32_t)c.b1, (uint32_t)inner); + } + case 12: { + GraphNode& anchor = *nodes[c.b4]; + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* res = nodes[c.a1]->output_buffer.get_data(); + void* ysum = nodes[c.a2]->output_buffer.get_data(); + const void* w = nodes[c.a3]->output_buffer.get_data(); + void* ynorm = anchor.output_buffer.get_data(); + if (!x || !res || !ysum || !w || !ynorm) return false; + return cactus_metal_encode_rms_norm_add_rows(ysum, ynorm, x, res, w, + c.u0, c.u1, c.f0, (int)c.b0); } default: return false; } diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 6a0ade2c2..2685e588c 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -148,7 +148,11 @@ bool cactus_metal_encode_groupnorm(void* out, const void* x, const void* w, cons bool cactus_metal_encode_bias_add_rows(void* y, const void* bias, uint32_t C, uint32_t total); bool cactus_metal_encode_elemwise_chain(void* out, const void* in, const float* steps, uint32_t nsteps, const void* side0, const void* side1, - const void* side2, size_t n, uint32_t flags); + const void* side2, const size_t* side_elems, + size_t n, uint32_t flags, uint32_t inner); +bool cactus_metal_encode_rms_norm_add_rows(void* ysum, void* ynorm, const void* x, const void* res, + const void* w, uint32_t rows, uint32_t dim, float eps, + int clipped); bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a); bool cactus_metal_encode_conv1d_dw(void* out, const void* x, const void* w, diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index fdf3eaf07..1d32c8c89 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -1016,6 +1016,38 @@ kernel void swiglu_f16(device const half* gate [[buffer(0)]], device const half* y[i]=(half)((float)g2*(float)up[i]); } +kernel void rms_norm_add_simd_f16(device const half* x [[buffer(0)]], device const half* res [[buffer(1)]], + device const half* w [[buffer(2)]], device half* ysum [[buffer(3)]], + device half* ynorm [[buffer(4)]], constant uint& dim [[buffer(5)]], + constant float& eps [[buffer(6)]], constant uint& rows [[buffer(7)]], + constant uint& clipped [[buffer(8)]], + uint tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + uint row = tg * 4 + sgid; + if (row >= rows) return; + device const half* xr = x + (size_t)row * dim; + device const half* rr = res + (size_t)row * dim; + device half* os = ysum + (size_t)row * dim; + device half* on = ynorm + (size_t)row * dim; + float partial = 0; + for (uint i = lane; i < dim; i += 32) { + float v = (float)xr[i] + (float)rr[i]; + if (clipped) v = clamp(v, -65500.0f, 65500.0f); + half h = (half)v; + os[i] = h; + float f = (float)h; + partial += f * f; + } + partial += simd_shuffle_xor(partial, 16); + partial += simd_shuffle_xor(partial, 8); + partial += simd_shuffle_xor(partial, 4); + partial += simd_shuffle_xor(partial, 2); + partial += simd_shuffle_xor(partial, 1); + float inv = 1.0f / sqrt(partial / (float)dim + eps); + for (uint i = lane; i < dim; i += 32) on[i] = (half)((float)os[i] * inv * (float)w[i]); +} + kernel void rms_norm_simd_f16(device const half* in [[buffer(0)]], device const half* w [[buffer(1)]], device half* y [[buffer(2)]], constant uint& dim [[buffer(3)]], constant float& eps [[buffer(4)]], constant uint& rows [[buffer(5)]], @@ -2131,6 +2163,7 @@ kernel void elemwise_chain_f16(device const uchar* in [[buffer(0)]], device ucha device const uchar* s0 [[buffer(4)]], device const uchar* s1 [[buffer(5)]], device const uchar* s2 [[buffer(6)]], constant uint& n [[buffer(7)]], constant uint& flags [[buffer(8)]], + constant uint& inner [[buffer(9)]], uint gid [[thread_position_in_grid]]) { uint i0 = gid * 4; if (i0 >= n) return; @@ -2149,8 +2182,12 @@ kernel void elemwise_chain_f16(device const uchar* in [[buffer(0)]], device ucha bool rhs = (st.code & 16) != 0; bool sf32 = (st.code & 32) != 0; device const uchar* sp = slot == 0 ? s0 : slot == 1 ? s1 : s2; + uint bmode = (st.code >> 8) & 3u; for (uint q = 0; q < cnt; ++q) { - float o = sf32 ? ((device const float*)sp)[i0+q] : (float)((device const half*)sp)[i0+q]; + uint si = i0 + q; + if (bmode == 1u) si = si / inner; + else if (bmode == 2u) si = si % inner; + float o = sf32 ? ((device const float*)sp)[si] : (float)((device const half*)sp)[si]; float a = rhs ? o : xv[q], b = rhs ? xv[q] : o; float r; switch (op) { case 2: r=a-b; break; case 3: r=a*b; break; case 4: r=a/b; break; diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 047339784..0593c6058 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -56,7 +56,7 @@ id psoBilinear=nil, psoConv1dGen=nil, psoConv1dNlcDw=nil, psoConv2d=nil; id psoBatchnorm=nil, psoGroupnorm=nil, psoBiasAddRows=nil, psoEwChain=nil; id psoAttnFlash=nil, psoRmsSimd=nil, psoScatterRows=nil; - id psoTranspose2d=nil, psoBcastRows=nil; + id psoTranspose2d=nil, psoBcastRows=nil, psoRmsAddSimd=nil; id dummy=nil; bool ok = false; @@ -114,7 +114,7 @@ psoBiasAddRows=pso("bias_add_rows_f16"); psoEwChain=pso("elemwise_chain_f16"); psoAttnFlash=pso("attn_flash_f16"); psoRmsSimd=pso("rms_norm_simd_f16"); psoScatterRows=pso("strided_scatter_rows_f16"); psoTranspose2d=pso("transpose2d_f16"); - psoBcastRows=pso("bcast_binary_rows_f16"); + psoBcastRows=pso("bcast_binary_rows_f16"); psoRmsAddSimd=pso("rms_norm_add_simd_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; @@ -1736,7 +1736,8 @@ bool cactus_metal_encode_bias_add_rows(void* y, const void* bias, uint32_t C, ui } bool cactus_metal_encode_elemwise_chain(void* out, const void* in, const float* steps, uint32_t nsteps, const void* side0, const void* side1, - const void* side2, size_t n, uint32_t flags) { + const void* side2, const size_t* side_elems, + size_t n, uint32_t flags, uint32_t inner) { if (!ctx().ok || !ctx().psoEwChain || nsteps == 0 || nsteps > 12) return false; ensureEncoder(); uint32_t ns = nsteps, nn = (uint32_t)n; @@ -1745,14 +1746,32 @@ bool cactus_metal_encode_elemwise_chain(void* out, const void* in, const float* setBufAt(in, n*ein, 0); setBufAt(out, n*eout, 1); [g_enc setBytes:steps length:nsteps*16 atIndex:2]; [g_enc setBytes:&ns length:4 atIndex:3]; - setBufAt(side0 ? side0 : in, n*((flags & 4u) ? 4 : 2), 4); - setBufAt(side1 ? side1 : in, n*((flags & 8u) ? 4 : 2), 5); - setBufAt(side2 ? side2 : in, n*((flags & 16u) ? 4 : 2), 6); + setBufAt(side0 ? side0 : in, (side0 ? side_elems[0] : n)*((flags & 4u) ? 4 : 2), 4); + setBufAt(side1 ? side1 : in, (side1 ? side_elems[1] : n)*((flags & 8u) ? 4 : 2), 5); + setBufAt(side2 ? side2 : in, (side2 ? side_elems[2] : n)*((flags & 16u) ? 4 : 2), 6); [g_enc setBytes:&nn length:4 atIndex:7]; [g_enc setBytes:&flags length:4 atIndex:8]; + uint32_t innr = inner; + [g_enc setBytes:&innr length:4 atIndex:9]; [g_enc dispatchThreads:MTLSizeMake((nn+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } + +bool cactus_metal_encode_rms_norm_add_rows(void* ysum, void* ynorm, const void* x, const void* res, + const void* w, uint32_t rows, uint32_t dim, float eps, + int clipped) { + if (!ctx().ok || !ctx().psoRmsAddSimd) return false; + ensureEncoder(); + uint32_t cl = clipped ? 1u : 0u; + [g_enc setComputePipelineState:ctx().psoRmsAddSimd]; + setBufAt(x, (size_t)rows*dim*2, 0); setBufAt(res, (size_t)rows*dim*2, 1); + setBufAt(w, (size_t)dim*2, 2); setBufAt(ysum, (size_t)rows*dim*2, 3); + setBufAt(ynorm, (size_t)rows*dim*2, 4); + [g_enc setBytes:&dim length:4 atIndex:5]; [g_enc setBytes:&eps length:4 atIndex:6]; + [g_enc setBytes:&rows length:4 atIndex:7]; [g_enc setBytes:&cl length:4 atIndex:8]; + [g_enc dispatchThreadgroups:MTLSizeMake((rows+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + return true; +} bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a) { if (!ctx().ok || !ctx().psoGemmBatch || !ctx().psoGemmBatchF32A) return false; diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index f3fef8487..a774a8c27 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -87,7 +87,8 @@ bool cactus_metal_encode_conv2d(void*, const void*, const void*, const void*, ui bool cactus_metal_encode_batchnorm(void*, const void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, float) { return false; } bool cactus_metal_encode_groupnorm(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, float) { return false; } bool cactus_metal_encode_bias_add_rows(void*, const void*, uint32_t, uint32_t) { return false; } -bool cactus_metal_encode_elemwise_chain(void*, const void*, const float*, uint32_t, const void*, const void*, const void*, size_t, uint32_t) { return false; } +bool cactus_metal_encode_elemwise_chain(void*, const void*, const float*, uint32_t, const void*, const void*, const void*, const size_t*, size_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_rms_norm_add_rows(void*, void*, const void*, const void*, const void*, uint32_t, uint32_t, float, int) { return false; } bool cactus_metal_encode_gemm_batch(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, int, int) { return false; } bool cactus_metal_encode_conv1d_dw(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_transpose2d(void*, const void*, uint32_t, uint32_t, uint32_t) { return false; } From 36a40e641c44585ae010ce3f4edaef1e0c49a9c3 Mon Sep 17 00:00:00 2001 From: jakmro Date: Fri, 3 Jul 2026 01:18:13 +0200 Subject: [PATCH 23/36] Review pass: close latent correctness holes and remove dead Metal paths Broadcast NOT_EQUAL now computes != in both bcast kernels instead of falling through to add. The disable-and-reexecute failure paths restore KV cache lengths so appends cannot double-apply. Flash attention edge tiles stage Q/O in an explicitly shared K/V pool instead of aliasing past the K array. The plan cache signature covers node count and op types, resets and the destructor invalidate plans and persistent activations, rule 12 gains the write-order guard rule 2 already had, rule 8 refuses retyped parents and anchors, and the rule 3 scale must be a graph input. Removed the dead rope_f16/lmhead_rotate encode chains, the unreachable executor branches, and leftover scaffolding; unified absent-bias binding on the dummy buffer and vectorized unary_f32 like its siblings. Parity suite grows a broadcast case and fails loudly on empty Metal output. Signed-off-by: jakmro --- cactus-engine/src/model.cpp | 3 +- cactus-graph/cactus_graph.h | 10 +++- cactus-graph/src/execute.cpp | 55 ++++++++++++++----- cactus-graph/src/metal_plan.cpp | 41 +++++++------- cactus-graph/src/metal_runtime.cpp | 24 +++++---- cactus-graph/tests/test_metal_parity.cpp | 20 ++++++- cactus-kernels/metal_backend.h | 2 - cactus-kernels/src/cactus_kernels.metal | 60 ++++++++------------- cactus-kernels/src/dsp.cpp | 8 ++- cactus-kernels/src/metal_backend.mm | 65 ++++++++--------------- cactus-kernels/src/metal_backend_stub.cpp | 2 - python/cactus/cli/run.py | 2 +- 12 files changed, 155 insertions(+), 137 deletions(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 52fb22e7b..9628b08b3 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -602,8 +602,7 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, bound.insert(comp); } - if (decoder_prefill_ && decoder_prefill_->graph - ) { + if (decoder_prefill_ && decoder_prefill_->graph) { try { if (prefill_encoder_ && prefill_encoder_->graph) { for (auto& b : prefill_encoder_->input_buffers) std::fill(b.begin(), b.end(), 0); diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index 1701cbbbd..fb6dfe27d 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -464,6 +464,11 @@ struct FusedEmbedCtx { bool ok = false; }; void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx); +const FusedEmbedCtx* cactus_graph_fused_embed(); +bool cactus_graph_metal_fold_prologue(void* h_buf, void* ple_buf, void* pos_buf, + const CactusQuantMatrix* lm_head, size_t nl, size_t ple_dim); +bool cactus_graph_metal_tail(void* logits, size_t vocab); +void cactus_graph_metal_tail_commit(); bool cactus_graph_metal_argmax(uint32_t* idx, float* best, float* second); void cactus_graph_mark_unadjusted(); @@ -730,8 +735,6 @@ class CactusGraph { void invalidate_persistent(size_t persistent_node_id); void execute(const std::string& profile_file = ""); - void build_metal_retype_plan(); - std::unordered_map metal_plans_; bool extract_ple_pathway(FusedEmbedCtx& ctx) const; void hard_reset(); void soft_reset(); @@ -785,6 +788,9 @@ class CactusGraph { std::unordered_set populated_node_ids_; std::unordered_set embedded_input_node_ids_; std::unordered_set retained_output_node_ids_; + void build_metal_retype_plan(); + void invalidate_metal_state(); + std::unordered_map metal_plans_; std::vector metal_retype_plan_; bool metal_retype_built_ = false; bool metal_retype_disabled_ = false; diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 579707939..4cdac312a 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -78,7 +78,6 @@ DECLARE_COMPUTE(compute_slice_node); DECLARE_COMPUTE(compute_embedding_node); DECLARE_COMPUTE(compute_concat_node); DECLARE_COMPUTE(compute_cat_node); -bool cactus_kv_cache_grow(BufferDesc&, size_t, size_t); DECLARE_COMPUTE(compute_index_node); DECLARE_COMPUTE(compute_bilinear_interpolation_node); DECLARE_COMPUTE(compute_sample_node); @@ -455,15 +454,12 @@ static void bcast_strides(const std::vector& in_shape, const std::vector out[d] = (d < off) ? 0u : (in_shape[d-off]==1 ? 0u : (uint32_t)istr[d-off]); } +bool cactus_kv_cache_grow(BufferDesc&, size_t, size_t); + static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const node_index_map_t& map) { BufferDesc& out = node.output_buffer; auto fp16 = [](const BufferDesc& b){ return b.precision == Precision::FP16; }; switch (node.op_type) { - case OpType::VIEW: case OpType::RESHAPE: case OpType::FLATTEN: { - const auto& in = get_input(node, 0, nodes, map); - if (in.byte_size != out.byte_size) return false; - return cactus_metal_encode_copy(out.get_data(), in.get_data(), in.byte_size); - } case OpType::MATMUL: { const auto& lhs = get_input(node, 0, nodes, map); const auto& rhs = get_input(node, 1, nodes, map); @@ -561,11 +557,14 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n size_t axis = (size_t)node.params.axis; uint32_t nd = (uint32_t)out.shape.size(); if (axis >= nd || nd > 8) return false; + for (size_t ii = 0; ii < node.input_ids.size(); ++ii) { + const auto& cin = get_input(node, ii, nodes, map); + if (!fp16(cin) || cin.shape.size() != nd) return false; + } size_t ostr[8]; row_strides(out.shape, ostr); size_t axis_off = 0; for (size_t ii = 0; ii < node.input_ids.size(); ++ii) { const auto& cin = get_input(node, ii, nodes, map); - if (!fp16(cin) || cin.shape.size() != nd) return false; uint32_t ishape[8], ostride[8]; for (uint32_t d=0; d& pinned_ids, const std::vector& retype); void cactus_metal_plan_free(MetalFusePlan* p); -void cactus_graph_metal_tail_commit(); void cactus_metal_plan_disable(MetalFusePlan* p); bool cactus_metal_plan_fold(MetalFusePlan* p, const std::vector>& nodes); const std::vector* cactus_metal_plan_exec_list(const MetalFusePlan* p); @@ -1225,7 +1223,10 @@ void CactusGraph::execute(const std::string& profile_file) { MetalFusePlan* fplan = nullptr; if (metal_mode && !need_debug) { uint64_t sig = 1469598103934665603ull; + sig ^= (uint64_t)n; sig *= 1099511628211ull; for (auto& np : nodes_) { + sig ^= (uint64_t)np->op_type + 0x9e3779b97f4a7c15ull; + sig *= 1099511628211ull; if (np->op_type != OpType::INPUT) continue; for (size_t d : np->output_buffer.shape) { sig ^= (uint64_t)d + 0x9e3779b97f4a7c15ull; @@ -1333,6 +1334,21 @@ void CactusGraph::execute(const std::string& profile_file) { size_t since_flush = 0; if (fplan) cactus_metal_plan_fold(fplan, nodes_); std::vector metal_live(n, 0); + std::vector> kv_snapshot; + for (size_t i = 0; i < n; ++i) { + const GraphNode& nd = *nodes_[i]; + if (nd.op_type != OpType::KV_CACHE_APPEND || nd.input_ids.size() < 2) continue; + auto ci = node_index_map_.find(nd.input_ids[1]); + if (ci == node_index_map_.end()) continue; + const uint64_t* km = reinterpret_cast(nodes_[ci->second]->output_buffer.get_data()); + if (km) kv_snapshot.push_back({ci->second, km[0]}); + } + auto kv_restore = [&]() { + for (const auto& s : kv_snapshot) { + uint64_t* km = reinterpret_cast(nodes_[s.first]->output_buffer.get_data()); + if (km) km[0] = s.second; + } + }; auto input_metal_live = [&](const GraphNode& nd) -> bool { for (size_t id : nd.input_ids) { auto it = node_index_map_.find(id); @@ -1355,10 +1371,6 @@ void CactusGraph::execute(const std::string& profile_file) { continue; } int32_t fact = fplan ? cactus_metal_plan_action(fplan, i) : -1; - if (fact == -2) { - for (size_t r : release_after[i]) metal_release(r); - continue; - } if (fact == -3) { assign_persistent_act(*node); metal_live[i] = 1; @@ -1394,6 +1406,7 @@ void CactusGraph::execute(const std::string& profile_file) { } cactus_metal_plan_disable(fplan); cactus_metal_session_sync(); + kv_restore(); cactus_metal_set_active(false); cactus_metal_session_end(); execute(profile_file); @@ -1410,13 +1423,15 @@ void CactusGraph::execute(const std::string& profile_file) { if (it != node_index_map_.end()) flip(nodes_[it->second]->output_buffer); } } - bool force_cpu = ot == OpType::PRECISION_CAST - && node->output_buffer.total_size <= 8 && !input_metal_live(*node); + bool force_cpu = (ot == OpType::PRECISION_CAST + && node->output_buffer.total_size <= 8 && !input_metal_live(*node)) + || (ot == OpType::EMBEDDING && input_metal_live(*node)); bool encoded = !force_cpu && try_encode_metal(*node, nodes_, node_index_map_); for (auto& fp : flipped) fp.first->precision = fp.second; if (!encoded && rplan && rplan[i] == 2) { metal_retype_disabled_ = true; cactus_metal_session_sync(); + kv_restore(); cactus_metal_set_active(false); cactus_metal_session_end(); execute(profile_file); @@ -1846,7 +1861,17 @@ void CactusGraph::execute(const std::string& profile_file) { } } +void CactusGraph::invalidate_metal_state() { + for (auto& kv : metal_plans_) cactus_metal_plan_free(kv.second); + metal_plans_.clear(); + for (auto& kv : metal_persistent_acts_) cactus_metal_free_shared(kv.second.first); + metal_persistent_acts_.clear(); + metal_retype_plan_.clear(); + metal_retype_built_ = false; +} + void CactusGraph::hard_reset() { + invalidate_metal_state(); nodes_.clear(); node_index_map_.clear(); mapped_files_.clear(); @@ -1857,6 +1882,7 @@ void CactusGraph::hard_reset() { } void CactusGraph::soft_reset() { + invalidate_metal_state(); std::set cached_node_ids; for (const auto& cache_entry : weight_cache_) { cached_node_ids.insert(cache_entry.second); @@ -1898,6 +1924,7 @@ void CactusGraph::soft_reset() { } void CactusGraph::soft_reset_keep_pool() { + invalidate_metal_state(); std::set cached_node_ids; for (const auto& cache_entry : weight_cache_) { cached_node_ids.insert(cache_entry.second); diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index e9900d040..e25851fc4 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -4,7 +4,6 @@ #include #include #include -#include namespace { @@ -39,14 +38,7 @@ struct MetalCluster { void* sc[3] = {nullptr, nullptr, nullptr}; }; -bool cactus_graph_metal_tail(void* logits, size_t vocab); -struct FusedEmbedCtx; -const FusedEmbedCtx* cactus_graph_fused_embed(); -bool cactus_graph_metal_fold_prologue(void* h_buf, void* ple_buf, void* pos_buf, - const CactusQuantMatrix* lm_head, size_t nl, size_t ple_dim); - struct MetalFusePlan { - bool built = false; std::vector action; std::vector clusters; long fold_h = -1, fold_ple = -1, fold_pos = -1, fold_w = -1; @@ -71,7 +63,6 @@ MetalFusePlan* cactus_metal_plan_build( const std::vector& retype) { const size_t n = nodes.size(); auto plan = new MetalFusePlan(); - plan->built = true; plan->action.assign(n, -1); std::vector pinned(n, 0); for (size_t i = 0; i < n; ++i) @@ -85,6 +76,7 @@ MetalFusePlan* cactus_metal_plan_build( for (size_t i = 0; i < n; ++i) for (size_t id : nodes[i]->input_ids) { long j = idxof(id); if (j >= 0) cons[(size_t)j].push_back(i); } + auto retyped = [&](size_t idx) { return idx < retype.size() && retype[idx] != 0; }; auto passthrough = [&](const GraphNode& nd) { return is_alias_op(nd.op_type) || is_same_cast(nd, nodes, map) || is_noop_transpose(nd); }; @@ -133,8 +125,6 @@ MetalFusePlan* cactus_metal_plan_build( if (!ok) { const GraphNode& cn = *nodes[c]; if (is_alias_op(cn.op_type) || is_same_cast(cn, nodes, map) || is_noop_transpose(cn)) { - std::vector sub = allowed; - sub.push_back(c); bool subok = true; for (size_t cc : cons[c]) { bool found = false; @@ -474,6 +464,7 @@ MetalFusePlan* cactus_metal_plan_build( { size_t ax = (size_t)nd.params.axis; const auto& osh = nd.output_buffer.shape; + if (retyped(i)) blocky = false; if (ax >= osh.size()) blocky = false; for (size_t d = 0; blocky && d < ax; ++d) if (osh[d] != 1) blocky = false; long fi = idxof(nd.input_ids[0]); @@ -540,6 +531,7 @@ MetalFusePlan* cactus_metal_plan_build( Precision wantA = jf32 ? Precision::FP32 : Precision::FP16; if (pa < 0 || pb < 0 || nodes[(size_t)pa]->output_buffer.precision != wantA + || (jf32 && retyped((size_t)pa)) || nodes[(size_t)pb]->output_buffer.precision != Precision::FP16) { ok = false; break; } if (j == 0) { parentA = pa; parentB = pb; offA0 = offA; offB0 = offB; f32a = jf32; } else if (jf32 != f32a) { ok = false; break; } @@ -724,6 +716,7 @@ MetalFusePlan* cactus_metal_plan_build( long scn = (o0 == (long)i || (o0 >= 0 && up(o0) == (long)i)) ? o1 : o0; long base = scn == o1 ? o0 : o1; if (base >= 0 && scn >= 0 + && nodes[(size_t)scn]->op_type == OpType::INPUT && nodes[(size_t)scn]->output_buffer.total_size == 1 && nodes[(size_t)scn]->output_buffer.precision == Precision::FP16) { c.b1 = (size_t)scn; @@ -923,7 +916,6 @@ MetalFusePlan* cactus_metal_plan_build( } }; auto prec_ok = [](Precision p) { return p == Precision::FP16 || p == Precision::FP32; }; - auto retyped = [&](size_t idx) { return idx < retype.size() && retype[idx] != 0; }; for (size_t i = 0; i < n; ++i) { if (plan->action[i] != -1) continue; GraphNode& nd0 = *nodes[i]; @@ -952,6 +944,10 @@ MetalFusePlan* cactus_metal_plan_build( || nodes[(size_t)x1]->output_buffer.total_size != an.output_buffer.total_size) continue; long ww = deep_f16_source(wi); if (ww < 0 || nodes[(size_t)ww]->output_buffer.total_size != dim) continue; + bool ordered = true; + for (size_t cc : cons[(size_t)ai]) + if (cc < i) { ordered = false; break; } + if (!ordered) continue; MetalCluster c; c.rule = 12; c.a0 = (size_t)x0; c.a1 = (size_t)x1; c.a2 = (size_t)ai; c.a3 = (size_t)ww; @@ -1005,12 +1001,6 @@ MetalFusePlan* cactus_metal_plan_build( long prev = ci == 0 ? -1 : (long)chain[ci - 1]; if (ci == 0) { long hi2 = idxof(cn.input_ids[0]); - if (k2 == 2) { - long in0 = idxof(cn.input_ids[0]); - long in1 = idxof(cn.input_ids[1]); - hi2 = in0; - (void)in1; - } if (hi2 < 0 || !prec_ok(nodes[(size_t)hi2]->output_buffer.precision) || retyped((size_t)hi2) || nodes[(size_t)hi2]->output_buffer.total_size != total) { ok = false; break; } @@ -1176,11 +1166,24 @@ MetalFusePlan* cactus_metal_plan_build( return plan; } -void cactus_metal_plan_free(MetalFusePlan* p) { delete p; } +static void release_cluster_buffers(MetalFusePlan* p) { + for (auto& c : p->clusters) { + if (c.s0) { cactus_metal_free_shared(c.s0); c.s0 = nullptr; } + if (c.s1) { cactus_metal_free_shared(c.s1); c.s1 = nullptr; } + for (auto& s : c.sc) if (s) { cactus_metal_free_shared(s); s = nullptr; } + } +} +void cactus_metal_plan_free(MetalFusePlan* p) { + if (!p) return; + release_cluster_buffers(p); + if (p->arena_base) { cactus_metal_free_shared(p->arena_base); p->arena_base = nullptr; } + delete p; +} void cactus_metal_plan_disable(MetalFusePlan* p) { if (!p) return; + release_cluster_buffers(p); std::fill(p->action.begin(), p->action.end(), -1); p->clusters.clear(); p->fold_h = -1; diff --git a/cactus-graph/src/metal_runtime.cpp b/cactus-graph/src/metal_runtime.cpp index 4ed7ee245..311dec03a 100644 --- a/cactus-graph/src/metal_runtime.cpp +++ b/cactus-graph/src/metal_runtime.cpp @@ -2,11 +2,7 @@ #include "cactus_kernels.h" #include "metal_backend.h" -struct MetalFusePlan; -void cactus_metal_plan_free(MetalFusePlan* p); -#include #include -#include #include #include @@ -62,14 +58,13 @@ static bool g_plan_tail_pending = false; static bool g_plan_tail_adjusted = false; static bool g_plan_tail_biased = false; -bool cactus_graph_metal_tail(void* logits, size_t vocab); -void cactus_graph_metal_tail_commit(); static const float* update_bias_dense(size_t vocab) { if (!g_samp.bias_dense || g_samp.bias_len == 0) return nullptr; if (!g_bias_dense || g_bias_dense_n < vocab) { + if (g_bias_dense) cactus_metal_free_shared(g_bias_dense); g_bias_dense = (float*)cactus_metal_alloc_shared(vocab * sizeof(float)); - if (!g_bias_dense) return nullptr; + if (!g_bias_dense) { g_bias_dense_n = 0; return nullptr; } g_bias_dense_n = vocab; } size_t n = g_samp.bias_len < vocab ? g_samp.bias_len : vocab; @@ -132,13 +127,22 @@ bool cactus_graph_metal_fold_prologue(void* h_buf, void* ple_buf, void* pos_buf, static void* pj = nullptr; static void* pjs = nullptr; static size_t cap = 0; + static size_t cap_pe = 0; const size_t PK = g_fe.proj.N; + const size_t EK = g_fe.ple.K; + if (cap_pe < EK) { + if (pe) cactus_metal_free_shared(pe); + pe = cactus_metal_alloc_shared(EK * 2); + cap_pe = pe ? EK : 0; + } if (cap < PK) { - pe = cactus_metal_alloc_shared(g_fe.ple.K * 2); + if (pa) cactus_metal_free_shared(pa); + if (pj) cactus_metal_free_shared(pj); + if (pjs) cactus_metal_free_shared(pjs); pa = cactus_metal_alloc_shared(PK * 2); pj = cactus_metal_alloc_shared(PK * 2); pjs = cactus_metal_alloc_shared(PK * 2); - cap = PK; + cap = (pa && pj && pjs) ? PK : 0; } if (!pe || !pa || !pj || !pjs) return false; uint32_t tok = (uint32_t)g_fe.token_id; @@ -188,5 +192,5 @@ void cactus_graph_on_destroy(const void* graph) { CactusGraph::~CactusGraph() { cactus_graph_on_destroy(this); - for (auto& kv : metal_plans_) cactus_metal_plan_free(kv.second); + invalidate_metal_state(); } diff --git a/cactus-graph/tests/test_metal_parity.cpp b/cactus-graph/tests/test_metal_parity.cpp index d43f24434..3181650b0 100644 --- a/cactus-graph/tests/test_metal_parity.cpp +++ b/cactus-graph/tests/test_metal_parity.cpp @@ -74,7 +74,6 @@ struct ParityCase { bool check() { std::vector cpu = run("cpu"); std::vector metal = run("metal"); - if (metal.empty()) return true; if (cpu.size() != metal.size() || cpu.empty()) return false; for (size_t i = 0; i < cpu.size(); ++i) { float scale = std::max(1.0f, std::fabs(cpu[i])); @@ -126,6 +125,24 @@ bool parity_not_equal() { return c.check(); } +bool parity_broadcast_binary() { + { + ParityCase c; + c.add_input({4, 33}); + c.add_input({1, 33}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.multiply(in[0], in[1]); }; + if (!c.check()) return false; + } + { + ParityCase c; + c.add_input({4, 33}); + c.add_input({1, 33}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.not_equal(in[0], in[1]); }; + if (!c.check()) return false; + } + return true; +} + bool parity_concat() { for (int axis = 0; axis < 2; ++axis) { ParityCase c; @@ -288,6 +305,7 @@ int main() { runner.run_test("leaky_relu", parity_scalar(&CactusGraph::leaky_relu, 0.1f)); runner.run_test("scalar_not_equal", parity_scalar(&CactusGraph::scalar_not_equal, 0.0f)); runner.run_test("not_equal", parity_not_equal()); + runner.run_test("broadcast_binary", parity_broadcast_binary()); runner.run_test("sum", parity_reduce(&CactusGraph::sum)); runner.run_test("mean", parity_reduce(&CactusGraph::mean)); runner.run_test("variance", parity_reduce(&CactusGraph::variance)); diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 2685e588c..d7a0ffce7 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -40,8 +40,6 @@ bool cactus_metal_encode_binary(int op_type, void* out, const void* a, const voi bool cactus_metal_encode_scalar(int op_type, void* out, const void* in, size_t n, float param); bool cactus_metal_encode_unary(int op_type, void* out, const void* in, size_t n); bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, size_t n, float scale); -bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const void* sin, - uint32_t heads, uint32_t head_dim); bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps); bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* weight, const void* res, diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 1d32c8c89..13c231eb0 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -30,7 +30,6 @@ kernel void cq4_transform( uint T [[threads_per_threadgroup]], threadgroup float* z [[threadgroup(0)]]) { - for (uint k=t; k=K) return; - float acc = 0; - for (uint k=0; k=total) return; - uint d=gid%hd, h=gid/hd, hh=hd/2; - float rot = (d=n) return; float x=in[i], r; - if (op==0) r=gelu_tanh(x); - else if (op==1) r=precise::tanh(x); - else if (op==2) r=x/(1.0f+precise::exp(-x)); - else if (op==4) r=0.5f*x*(1.0f+erf_approx(x*0.70710678f)); - else if (op==5) r=1.0f/(1.0f+precise::exp(-x)); - else r=max(x,0.0f); - y[i]=r; + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) { + float x=in[i0+j], r; + if (op==0) r=gelu_tanh(x); + else if (op==1) r=precise::tanh(x); + else if (op==2) r=x/(1.0f+precise::exp(-x)); + else if (op==4) r=0.5f*x*(1.0f+erf_approx(x*0.70710678f)); + else if (op==5) r=1.0f/(1.0f+precise::exp(-x)); + else r=max(x,0.0f); + y[i0+j]=r; + } } kernel void clamp_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], @@ -1656,8 +1639,9 @@ kernel void attn_flash_f16(device const half* q [[buffer(0)]], device const half uint sgid [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]]) { constexpr uint BQ = 64, BK = 32, D = 64, NT = 256; - threadgroup half Ks[BK * D]; - threadgroup half Vs[BK * D]; + threadgroup half KVs[2 * BK * D]; + threadgroup half* Ks = KVs; + threadgroup half* Vs = KVs + BK * D; threadgroup float Ss[8][8 * BK]; threadgroup half Ps[8][8 * BK]; threadgroup half Dg[8][64]; @@ -1678,7 +1662,7 @@ kernel void attn_flash_f16(device const half* q [[buffer(0)]], device const half for (uint d = 0; d < 8; ++d) simdgroup_load(qf[d], q + ((size_t)row0 * H + h) * D + d * 8, (size_t)H * D); } else { - threadgroup half* Qe = Ks; + threadgroup half* Qe = KVs; for (uint x = tid; x < BQ * D; x += NT) { uint r = x / D, d = x % D; uint t = q0 + r; @@ -1797,7 +1781,7 @@ kernel void attn_flash_f16(device const half* q [[buffer(0)]], device const half simdgroup_store(of[d], y + ((size_t)row0 * H + h) * D + d * 8, (size_t)H * D); } } else { - threadgroup half* Oe = Ks; + threadgroup half* Oe = KVs; for (uint d = 0; d < 8; ++d) { simdgroup_multiply(of[d], dgf, of[d]); simdgroup_store(of[d], Oe + (size_t)sgid * 8 * D + d * 8, D); @@ -1981,7 +1965,6 @@ kernel void rope_full_f16(device const half* x [[buffer(0)]], device half* y [[b uint d = gid.x, tok = gid.y; if (d >= D) return; size_t off = (size_t)tok * D + d; - uint hd = D; uint seq = (tok / H) % S; if (d >= rot) { y[off] = x[off]; return; } uint hr = rot / 2; @@ -1996,7 +1979,6 @@ kernel void rope_full_f16(device const half* x [[buffer(0)]], device half* y [[b if (gptj) { first = (d % 2) == 0; other = (float)x[off + (first ? 1 : -1)]; } else { first = d < hr; other = (float)x[off + (first ? (long)hr : -(long)hr)]; } y[off] = (half)(first ? v*c - other*sn : v*c + other*sn); - (void)hd; } kernel void maxpool1d_f16(device const half* x [[buffer(0)]], device half* y [[buffer(1)]], @@ -2228,7 +2210,8 @@ kernel void bcast_binary_rows_f16(device const half* a [[buffer(0)]], device con uint rem = row, ai = 0, bi = 0; for (int d = int(ndim) - 1; d >= 0; --d) { uint c = rem % oshape[d]; rem /= oshape[d]; ai += c * astride[d]; bi += c * bstride[d]; } float av = (float)a[ai + v * ainner], bv = (float)b[bi + v * binner], r; - switch(op){ case 2:r=av-bv;break; case 3:r=av*bv;break; case 4:r=av/bv;break; default:r=av+bv; } + switch(op){ case 2:r=av-bv;break; case 3:r=av*bv;break; case 4:r=av/bv;break; + case 5:r=(av!=bv)?1.0f:0.0f;break; default:r=av+bv; } if (op==1) r=clamp(r,-65500.0f,65500.0f); out[(size_t)row * inner + v] = (half)r; } @@ -2242,7 +2225,8 @@ kernel void bcast_binary_f16(device const half* a [[buffer(0)]], device const ha uint rem=i, ai=0, bi=0; for (int d=int(ndim)-1; d>=0; --d){ uint c=rem%oshape[d]; rem/=oshape[d]; ai+=c*astride[d]; bi+=c*bstride[d]; } float av=(float)a[ai], bv=(float)b[bi], r; - switch(op){ case 2:r=av-bv;break; case 3:r=av*bv;break; case 4:r=av/bv;break; default:r=av+bv; } + switch(op){ case 2:r=av-bv;break; case 3:r=av*bv;break; case 4:r=av/bv;break; + case 5:r=(av!=bv)?1.0f:0.0f;break; default:r=av+bv; } if (op==1) r=clamp(r,-65500.0f,65500.0f); out[i]=(half)r; } diff --git a/cactus-kernels/src/dsp.cpp b/cactus-kernels/src/dsp.cpp index bee00e0af..bfe01eb76 100644 --- a/cactus-kernels/src/dsp.cpp +++ b/cactus-kernels/src/dsp.cpp @@ -158,8 +158,12 @@ void cactus_rfft_f32_1d(const float* input, float* output, size_t n, const char* { std::lock_guard lk(fft_mu); auto it = fft_cache.find(log2n); - if (it == fft_cache.end()) it = fft_cache.emplace(log2n, vDSP_create_fftsetup(log2n, FFT_RADIX2)).first; - fft_setup = it->second; + if (it == fft_cache.end()) { + FFTSetup created = vDSP_create_fftsetup(log2n, FFT_RADIX2); + if (created) it = fft_cache.emplace(log2n, created).first; + else it = fft_cache.end(); + } + fft_setup = it != fft_cache.end() ? it->second : nullptr; } if (fft_setup) { std::vector buffer(padded_n, 0.0f); diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 0593c6058..2a037ac75 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -34,7 +33,7 @@ id psoActQ = nil, psoG2i8 = nil; id psoTm = nil, psoGm = nil; id psoGmma = nil, psoGdense = nil, psoDeq = nil; - id psoRotate = nil, psoRotW = nil, psoEmbOW = nil, psoRmsAddRms = nil; + id psoRotW = nil, psoEmbOW = nil, psoRmsAddRms = nil; id psoEmbO = nil, psoEmbH = nil; id psoEmbOm = nil, psoEmbHm = nil; id psoCopy=nil, psoBinary=nil, psoScalar=nil, psoUnary=nil, psoRms=nil, psoSwiglu=nil, psoRmsAdd=nil, psoRmsAddScale=nil; @@ -43,7 +42,6 @@ id psoAttnPre=nil, psoAttnPreMma2=nil, psoAttnPreHd256=nil; id psoKvAppendM=nil, psoKvAppendRingM=nil; id psoSlideS=nil, psoSlideR=nil, psoSlideRM=nil; - id psoRope=nil; id psoArgmax=nil; id psoArgmaxP=nil, psoArgmaxF=nil, psoSoftcap=nil, psoAdjust=nil; id psoGather=nil; @@ -81,7 +79,7 @@ psoGdense=pso("cq4_gemm_dense_f16"); psoDeq=pso("cq4_dequant_f16"); psoGmrLow=pso("cq_gemv_mr_lowbit"); psoDeqLow=pso("cq_dequant_lowbit_f16"); psoActQ=pso("cq_act_quant_i8"); psoG2i8=pso("cq2_gemv_i8"); - psoRotate=pso("lmhead_rotate"); psoRotW=pso("lmhead_rotate_wide"); psoEmbOW=pso("emb_ortho_wide"); psoRmsAddRms=pso("rms_norm_add_rms_f16"); + psoRotW=pso("lmhead_rotate_wide"); psoEmbOW=pso("emb_ortho_wide"); psoRmsAddRms=pso("rms_norm_add_rms_f16"); psoEmbO=pso("emb_ortho"); psoEmbH=pso("emb_hadamard"); psoEmbOm=pso("emb_ortho_m"); psoEmbHm=pso("emb_hadamard_m"); psoGather=pso("gather_f16"); @@ -94,7 +92,7 @@ psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoAttnPreHd256=pso("attn_prefill_mma_hd256"); psoKvAppendM=pso("kv_append_i8_m"); psoKvAppendRingM=pso("kv_append_ring_i8_m"); psoSlideS=pso("kv_slide_save"); psoSlideR=pso("kv_slide_restore"); psoSlideRM=pso("kv_slide_restore_m"); - psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoRope=pso("rope_f16"); + psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); psoArgmax=pso("argmax_logits"); psoArgmaxP=pso("argmax_part"); psoArgmaxF=pso("argmax_final"); psoSoftcap=pso("softcap_f16"); psoAdjust=pso("adjust_logits"); @@ -116,8 +114,8 @@ psoScatterRows=pso("strided_scatter_rows_f16"); psoTranspose2d=pso("transpose2d_f16"); psoBcastRows=pso("bcast_binary_rows_f16"); psoRmsAddSimd=pso("rms_norm_add_simd_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; - ok = psoT&&psoG&&psoTm&&psoGm&&psoRotate&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 - &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoRope&&psoArgmax&&psoGather; + ok = psoT&&psoG&&psoTm&&psoGm&&psoRotW&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 + &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoArgmax&&psoGather; }} }; @@ -473,18 +471,6 @@ bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, siz [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } -bool cactus_metal_encode_rope(void* out, const void* x, const void* cos, const void* sin, - uint32_t heads, uint32_t head_dim) { - if (!ctx().ok) return false; - ensureEncoder(); - uint32_t total=heads*head_dim; - [g_enc setComputePipelineState:ctx().psoRope]; - setBufAt(x, (size_t)total*2, 0); setBufAt(out, (size_t)total*2, 1); - setBufAt(cos, (size_t)head_dim*2, 2); setBufAt(sin, (size_t)head_dim*2, 3); - [g_enc setBytes:&heads length:4 atIndex:4]; [g_enc setBytes:&head_dim length:4 atIndex:5]; - [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - return true; -} bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps) { if (!ctx().ok) return false; @@ -660,7 +646,7 @@ void cactus_metal_trim_prefill_cache() { bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W) { if (!ctx().ok || !W) return false; const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; - if (ctx().ok && (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->rotation && ng==1 && gs==K && (N%4)==0) { + if ((W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->rotation && ng==1 && gs==K && (N%4)==0) { static void* ortho_code = nullptr; static uint32_t ortho_code_k = 0; if (ortho_code_k < K) { if (ortho_code) cactus_metal_free_shared(ortho_code); @@ -669,8 +655,8 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu } if (ortho_code) return cactus_metal_encode_quant_matmul_ortho(out, lhs, ortho_code, W); } - const bool fast = ctx().ok && quant_fast_eligible(W); - const bool lowbit = !fast && ctx().ok && ctx().psoGmrLow && quant_lowbit_eligible(W); + const bool fast = quant_fast_eligible(W); + const bool lowbit = !fast && ctx().psoGmrLow && quant_lowbit_eligible(W); if (!fast && !lowbit) return false; const uint32_t bits = W->bits, pgbw = (gs*bits+7u)/8u; ResW& rw = resident(W); @@ -910,20 +896,12 @@ bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* co if (!rw.rotation || !rw.packed) return false; ensureEncoder(); - if (ctx().psoRotW) { - [g_enc setComputePipelineState:ctx().psoRotW]; - setBufAt(act, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; - [g_enc setBuffer:rw.rotation offset:0 atIndex:2]; setBufAt(code, (size_t)K*2, 3); - [g_enc setBytes:&K length:4 atIndex:4]; - [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake((K+31)/32,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - } else { - [g_enc setComputePipelineState:ctx().psoRotate]; - setBufAt(act, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; - [g_enc setBuffer:rw.rotation offset:0 atIndex:2]; setBufAt(code, (size_t)K*2, 3); - [g_enc setBytes:&K length:4 atIndex:4]; - [g_enc dispatchThreads:MTLSizeMake(K,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - } + [g_enc setComputePipelineState:ctx().psoRotW]; + setBufAt(act, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; + [g_enc setBuffer:rw.rotation offset:0 atIndex:2]; setBufAt(code, (size_t)K*2, 3); + [g_enc setBytes:&K length:4 atIndex:4]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((K+31)/32,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; bool umr = ctx().psoGmr && (N % g_mr_rows == 0u); [g_enc setComputePipelineState:(lowbit?ctx().psoGmrLow:(umr?ctx().psoGmr:ctx().psoG))]; @@ -1095,9 +1073,9 @@ bool cactus_metal_encode_attention_i8( float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes) { if (!ctx().ok) return false; if (kv_end <= kv_start || (num_q_heads % num_kv_heads) != 0) return false; + if (head_dim > 512u || v_hdim > 512u) return false; ensureEncoder(); auto setCache = [&](const void* p, size_t bytes, int idx) { - if (p && bytes) setBufAt(p, bytes, idx); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:idx]; }; @@ -1201,9 +1179,8 @@ bool cactus_metal_encode_attention_i8_prefill( const uint32_t T = 128; if (total_keys == 0 || M == 0 || (num_q_heads % num_kv_heads) != 0) return false; uint32_t maxsc = (ring > 0u) ? ((total_keys > sink + ring) ? (sink + ring) : total_keys) : 256u; - bool mma2 = (ring == 0u && is_causal && total_keys >= 2048u && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u); - if (!mma2 && is_causal && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u - && window == 0u) mma2 = true; + bool mma2 = (ring == 0u && window == 0u && is_causal && head_dim == 512u && v_hdim == 512u + && num_q_heads == 8u && num_kv_heads == 1u); bool hd256 = (ctx().psoAttnPreHd256 != nil && is_causal && head_dim == 256u && v_hdim == 256u && num_q_heads == 8u && num_kv_heads == 1u); if (mma2 || hd256) { @@ -1284,7 +1261,7 @@ bool cactus_metal_encode_unary_f32(int op, void* out, const void* in, size_t n) [g_enc setComputePipelineState:ctx().psoUnaF32]; setBufAt(in, n*4, 0); setBufAt(out, n*4, 1); [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; - [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + [g_enc dispatchThreads:MTLSizeMake((n+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } bool cactus_metal_encode_clamp(void* out, const void* in, size_t n, float lo, float hi, int f32) { @@ -1651,7 +1628,7 @@ bool cactus_metal_encode_conv1d_gen(void* out, const void* x, const void* w, con uint32_t hb = bias ? 1u : 0u, wl = w_ck_co ? 1u : 0u; [g_enc setComputePipelineState:ctx().psoConv1dGen]; setBufAt(x, (size_t)N*Cin*L*2, 0); setBufAt(w, (size_t)Cout*Cin*K*2, 1); - setBufAt(bias ? bias : x, bias ? (size_t)Cout*2 : 2, 2); + if (bias) setBufAt(bias, (size_t)Cout*2, 2); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; setBufAt(out, (size_t)N*Cout*Lout*2, 3); [g_enc setBytes:&Cin length:4 atIndex:4]; [g_enc setBytes:&L length:4 atIndex:5]; [g_enc setBytes:&Cout length:4 atIndex:6]; [g_enc setBytes:&Lout length:4 atIndex:7]; @@ -1668,7 +1645,7 @@ bool cactus_metal_encode_conv1d_nlc_dw(void* out, const void* x, const void* w, uint32_t hb = bias ? 1u : 0u; [g_enc setComputePipelineState:ctx().psoConv1dNlcDw]; setBufAt(x, (size_t)N*L*C*2, 0); setBufAt(w, (size_t)C*K*2, 1); - setBufAt(bias ? bias : x, bias ? (size_t)C*2 : 2, 2); + if (bias) setBufAt(bias, (size_t)C*2, 2); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; setBufAt(out, (size_t)N*L*C*2, 3); [g_enc setBytes:&L length:4 atIndex:4]; [g_enc setBytes:&C length:4 atIndex:5]; [g_enc setBytes:&K length:4 atIndex:6]; [g_enc setBytes:&dil length:4 atIndex:7]; @@ -1686,7 +1663,7 @@ bool cactus_metal_encode_conv2d(void* out, const void* x, const void* w, const v size_t wbytes = dw ? (size_t)Cout*K*K*2 : (size_t)Cout*Cin*K*K*2; [g_enc setComputePipelineState:ctx().psoConv2d]; setBufAt(x, (size_t)N*Cin*H*W*2, 0); setBufAt(w, wbytes, 1); - setBufAt(bias ? bias : x, bias ? (size_t)Cout*2 : 2, 2); + if (bias) setBufAt(bias, (size_t)Cout*2, 2); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; setBufAt(out, (size_t)N*Cout*Ho*Wo*2, 3); [g_enc setBytes:&Cin length:4 atIndex:4]; [g_enc setBytes:&H length:4 atIndex:5]; [g_enc setBytes:&W length:4 atIndex:6]; [g_enc setBytes:&Cout length:4 atIndex:7]; diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index a774a8c27..ea1713f5a 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -31,8 +31,6 @@ bool cactus_metal_encode_binary(int, void*, const void*, const void*, size_t) { bool cactus_metal_encode_scalar(int, void*, const void*, size_t, float) { return false; } bool cactus_metal_encode_unary(int, void*, const void*, size_t) { return false; } bool cactus_metal_encode_swiglu(void*, const void*, const void*, size_t, float) { return false; } -bool cactus_metal_encode_rope(void*, const void*, const void*, const void*, uint32_t, uint32_t) { return false; } -bool cactus_metal_encode_rope_m(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_rms_norm(void*, const void*, const void*, size_t, size_t, float) { return false; } bool cactus_metal_encode_rms_norm_add(void*, const void*, const void*, const void*, size_t, size_t, float) { return false; } bool cactus_metal_encode_rms_norm_add_rms(void*, void*, const void*, const void*, const void*, const void*, diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index 98de2d6a9..38078a3a8 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -40,7 +40,7 @@ def cmd_run(args) -> int: ("--result-json", args.result_json), ("--confidence-threshold", args.confidence_threshold), ("--cloud-timeout-ms", args.cloud_timeout_ms), - ("--backend", getattr(args, "backend", "auto")), + ("--backend", args.backend), ): if value is not None: cmd.extend([flag, str(value)]) From 786f3cfa2615104ea91dd7d5903c07a0df510d26 Mon Sep 17 00:00:00 2001 From: jakmro Date: Fri, 3 Jul 2026 04:58:15 +0200 Subject: [PATCH 24/36] fixes Signed-off-by: jakmro --- cactus-engine/src/embed.cpp | 10 ++ cactus-engine/src/transcribe.cpp | 4 + cactus-graph/src/execute.cpp | 152 ++++++++++++++++++++-- cactus-graph/src/metal_plan.cpp | 70 ++++++++++ cactus-graph/src/ops_conv.cpp | 2 - cactus-kernels/metal_backend.h | 10 ++ cactus-kernels/src/cactus_kernels.metal | 64 +++++++++ cactus-kernels/src/metal_backend.mm | 90 ++++++++++++- cactus-kernels/src/metal_backend_stub.cpp | 6 + 9 files changed, 391 insertions(+), 17 deletions(-) diff --git a/cactus-engine/src/embed.cpp b/cactus-engine/src/embed.cpp index 6a380836a..33e9daf7f 100644 --- a/cactus-engine/src/embed.cpp +++ b/cactus-engine/src/embed.cpp @@ -1,6 +1,7 @@ #include "../cactus_engine.h" #include "utils.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include "wav.h" #include #include @@ -37,6 +38,9 @@ int cactus_embed( size_t* embedding_dim, bool normalize ) { + struct MetalTrimGuard { + ~MetalTrimGuard() { cactus_metal_trim_prefill_cache(); } + } metal_trim_guard; if (!model || !text || !embeddings_buffer || buffer_size == 0) { CACTUS_LOG_ERROR("embed", "Invalid parameters for text embedding"); return -1; @@ -90,6 +94,9 @@ int cactus_image_embed( size_t buffer_size, size_t* embedding_dim ) { + struct MetalTrimGuard { + ~MetalTrimGuard() { cactus_metal_trim_prefill_cache(); } + } metal_trim_guard; if (!model || !image_path || !embeddings_buffer || buffer_size == 0) { CACTUS_LOG_ERROR("image_embed", "Invalid parameters for image embedding"); return -1; @@ -130,6 +137,9 @@ int cactus_audio_embed( size_t buffer_size, size_t* embedding_dim ) { + struct MetalTrimGuard { + ~MetalTrimGuard() { cactus_metal_trim_prefill_cache(); } + } metal_trim_guard; if (!model || !audio_path || !embeddings_buffer || buffer_size == 0) { CACTUS_LOG_ERROR("audio_embed", "Invalid parameters for audio embedding"); return -1; diff --git a/cactus-engine/src/transcribe.cpp b/cactus-engine/src/transcribe.cpp index cafd98c67..9f2ebe928 100644 --- a/cactus-engine/src/transcribe.cpp +++ b/cactus-engine/src/transcribe.cpp @@ -2,6 +2,7 @@ #include "utils.h" #include "cloud.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include "wav.h" #include #include @@ -163,6 +164,9 @@ int cactus_transcribe( const uint8_t* pcm_buffer, size_t pcm_buffer_size ) { + struct MetalTrimGuard { + ~MetalTrimGuard() { cactus_metal_trim_prefill_cache(); } + } metal_trim_guard; if (validate_audio_params("transcribe", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) { return -1; diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 4cdac312a..cd29e22bb 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -455,6 +455,31 @@ static void bcast_strides(const std::vector& in_shape, const std::vector } bool cactus_kv_cache_grow(BufferDesc&, size_t, size_t); +std::vector<__fp16> dequantize_int8_weights_to_fp16(const BufferDesc& W, size_t rows, size_t cols, + const char* op_name); + +static const __fp16* metal_conv_weight_f16(const BufferDesc& w, size_t rows, size_t cols) { + if (w.precision == Precision::FP16) return w.data_as<__fp16>(); + if (w.precision != Precision::INT8) return nullptr; + const int8_t* p = w.data_as(); + size_t n = rows * cols; + if (!p || n == 0 || w.total_size < n) return nullptr; + uint64_t h = 1469598103934665603ull; + size_t take = n < 64 ? n : 64; + for (size_t i = 0; i < take; ++i) { h ^= (uint8_t)p[i]; h *= 1099511628211ull; } + for (size_t i = n - take; i < n; ++i) { h ^= (uint8_t)p[i]; h *= 1099511628211ull; } + h ^= n; h *= 1099511628211ull; + struct Entry { std::vector<__fp16> data; uint64_t fp; }; + static std::mutex mu; + static std::unordered_map cache; + std::lock_guard lk(mu); + Entry& e = cache[p]; + if (e.data.size() != n || e.fp != h) { + e.data = dequantize_int8_weights_to_fp16(w, rows, cols, "conv_metal"); + e.fp = h; + } + return e.data.data(); +} static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const node_index_map_t& map) { BufferDesc& out = node.output_buffer; @@ -712,7 +737,7 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n const auto& w = get_input(node, 1, nodes, map); bool causal = node.op_type == OpType::CONV1D_CAUSAL; const BufferDesc* b = (!causal && node.input_ids.size() > 2) ? &get_input(node, 2, nodes, map) : nullptr; - if (!fp16(x) || !fp16(w) || !fp16(out)) return false; + if (!fp16(x) || !fp16(out)) return false; if (b && b->precision != Precision::FP16) return false; if (x.shape.size() != 3) return false; size_t C = x.shape[2]; @@ -724,9 +749,11 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n K = w.shape.back(); if (w.total_size != C * K) return false; } + const __fp16* wptr = metal_conv_weight_f16(w, C, K); + if (!wptr) return false; size_t dil = causal ? (node.params.dilation ? node.params.dilation : 1) : 1; size_t pad = causal ? (K - 1) * dil : K / 2; - return cactus_metal_encode_conv1d_nlc_dw(out.get_data(), x.get_data(), w.get_data(), + return cactus_metal_encode_conv1d_nlc_dw(out.get_data(), x.get_data(), wptr, b ? b->get_data() : nullptr, (uint32_t)x.shape[0], (uint32_t)x.shape[1], (uint32_t)C, (uint32_t)K, (uint32_t)dil, (uint32_t)pad); } @@ -754,18 +781,35 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n const auto& x = get_input(node, 0, nodes, map); const auto& w = get_input(node, 1, nodes, map); const BufferDesc* b = node.input_ids.size() > 2 ? &get_input(node, 2, nodes, map) : nullptr; - if (!fp16(x) || !fp16(w) || !fp16(out)) return false; + if (!fp16(x) || !fp16(out)) return false; if (b && b->precision != Precision::FP16) return false; if (x.shape.size() != 3 || out.shape.size() != 3) return false; size_t Cin = x.shape[2], Cout = out.shape[2]; if (w.total_size != Cin * Cout) return false; + const __fp16* wptr = metal_conv_weight_f16(w, Cout, Cin); + if (!wptr) return false; size_t rows = x.shape[0] * x.shape[1]; - if (!cactus_metal_encode_gemm_f16(out.get_data(), x.get_data(), w.get_data(), + if (!cactus_metal_encode_gemm_f16(out.get_data(), x.get_data(), wptr, (uint32_t)rows, (uint32_t)Cin, (uint32_t)Cout, 1)) return false; if (b) return cactus_metal_encode_bias_add_rows(out.get_data(), b->get_data(), (uint32_t)Cout, (uint32_t)out.total_size); return true; } + case OpType::REL_POS_BIAS: { + if (node.input_ids.size() != 2) return false; + const auto& q = get_input(node, 0, nodes, map); + const auto& r = get_input(node, 1, nodes, map); + if (!fp16(q) || !fp16(r) || !fp16(out)) return false; + if (q.shape.size() != 4 || r.shape.size() != 4) return false; + size_t B = q.shape[0], T = q.shape[1], H = q.shape[2], D = q.shape[3]; + size_t Rb = r.shape[0], R = r.shape[1]; + if (B == 0 || T == 0 || (Rb != 1 && Rb != B)) return false; + if (r.shape[2] != H || r.shape[3] != D) return false; + if (R < 2 * T - 1 || out.total_size != B * H * T * T) return false; + return cactus_metal_encode_rel_pos_bias(out.get_data(), q.get_data(), r.get_data(), + (uint32_t)B, (uint32_t)T, (uint32_t)H, (uint32_t)D, (uint32_t)R, + Rb == 1 ? 0 : 1, node.params.scale); + } case OpType::BATCHNORM: { if (node.input_ids.size() != 5) return false; const auto& x = get_input(node, 0, nodes, map); @@ -1030,6 +1074,28 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n if (out.get_data()) *out.data_as() = static_cast(new_total); return true; } + case OpType::CONV_CACHE_APPEND: { + if (node.input_ids.size() < 2) return false; + const auto& new_data = get_input(node, 0, nodes, map); + BufferDesc& cache = nodes[map.at(node.input_ids[1])]->output_buffer; + if (!fp16(out) || !cache.get_data() || !out.get_data()) return false; + if (new_data.precision != Precision::FP16 && new_data.precision != Precision::FP32) return false; + uint64_t* cm = reinterpret_cast(cache.get_data()); + uint64_t head = cm[0], count = cm[1], ws = cm[2], hd = cm[3]; + if (ws == 0 || hd == 0 || head >= ws) return false; + size_t num_rows = new_data.total_size / hd; + if (num_rows == 0 || out.total_size != ws * hd) return false; + uint32_t nnew = (uint32_t)std::min(num_rows, ws); + uint64_t count_new = std::min(ws, count + num_rows); + if (!cactus_metal_encode_conv_cache_append(out.get_data(), new_data.get_data(), + static_cast(cache.get_data()) + 64, + (uint32_t)hd, (uint32_t)ws, nnew, (uint32_t)head, (uint32_t)count_new, + (uint32_t)num_rows, new_data.precision == Precision::FP32 ? 1 : 0)) + return false; + cm[0] = (head + nnew) % ws; + cm[1] = count_new; + return true; + } case OpType::EMBEDDING: { if (node.input_ids.size() < 2) return false; const auto& emb = get_input(node, 0, nodes, map); @@ -1068,6 +1134,8 @@ void cactus_metal_plan_disable(MetalFusePlan* p); bool cactus_metal_plan_fold(MetalFusePlan* p, const std::vector>& nodes); const std::vector* cactus_metal_plan_exec_list(const MetalFusePlan* p); void* cactus_metal_plan_arena_ptr(const MetalFusePlan* p, size_t i); +bool cactus_metal_plan_has_arena(const MetalFusePlan* p); +void cactus_metal_plan_extend_last_use(const MetalFusePlan* p, std::vector& last_use); int32_t cactus_metal_plan_action(const MetalFusePlan* p, size_t i); bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, const std::vector>& nodes, @@ -1218,7 +1286,16 @@ void CactusGraph::execute(const std::string& profile_file) { return true; }; - const bool metal_mode = cactus_backend_metal(); + bool metal_mode = cactus_backend_metal(); + if (metal_mode && n < 100) { + for (size_t i = 0; i < n; ++i) { + OpType t = nodes_[i]->op_type; + if (t == OpType::LSTM_CELL || t == OpType::BILSTM_SEQUENCE) { + metal_mode = false; + break; + } + } + } if (metal_mode && !need_debug && !metal_retype_built_) build_metal_retype_plan(); MetalFusePlan* fplan = nullptr; if (metal_mode && !need_debug) { @@ -1304,6 +1381,8 @@ void CactusGraph::execute(const std::string& profile_file) { } } + if (fplan) cactus_metal_plan_extend_last_use(fplan, last_use); + std::vector> release_after(n); for (size_t i = 0; i < n; ++i) { if (!can_release_node(i) || use_count[i] == 0 || keep_until_graph_cleanup[i]) continue; @@ -1313,9 +1392,32 @@ void CactusGraph::execute(const std::string& profile_file) { if (metal_mode && !need_debug) { cactus_metal_session_begin(); cactus_metal_set_active(true); + const bool transient_acts = n >= 1500 && !cactus_metal_plan_has_arena(fplan); + std::vector transient_ptr(transient_acts ? n : 0, nullptr); + std::vector transient_dead; + std::vector transient_ok(transient_acts ? n : 0, 0); + if (transient_acts) { + for (size_t i = 0; i < n; ++i) { + size_t id = nodes_[i]->id; + transient_ok[i] = !retained_output_node_ids_.count(id) + && !persistent_node_ids_.count(id) + && !keep_until_graph_cleanup[i] + && nodes_[i]->output_buffer.byte_size >= (256u << 10); + } + } + auto release_transients = [&]() { + for (void* p : transient_dead) cactus_metal_free_shared(p); + transient_dead.clear(); + for (size_t i = 0; i < transient_ptr.size(); ++i) + if (transient_ptr[i]) { cactus_metal_free_shared(transient_ptr[i]); transient_ptr[i] = nullptr; } + }; auto metal_release = [&](size_t idx) { GraphNode& nd = *nodes_[idx]; if (aliases_input(nd)) nd.output_buffer.external_data = nullptr; + if (transient_acts && transient_ptr[idx]) { + transient_dead.push_back(transient_ptr[idx]); + transient_ptr[idx] = nullptr; + } }; auto assign_persistent_act = [&](GraphNode& nd) { size_t need = nd.output_buffer.byte_size; @@ -1331,22 +1433,36 @@ void CactusGraph::execute(const std::string& profile_file) { if (p) { nd.output_buffer.release_to_pool(pool); nd.output_buffer.set_external(p); } else nd.output_buffer.resize_from_pool(pool); }; - size_t since_flush = 0; + size_t since_flush = 0, since_recycle = 0; if (fplan) cactus_metal_plan_fold(fplan, nodes_); std::vector metal_live(n, 0); - std::vector> kv_snapshot; + auto maybe_recycle = [&]() { + if (!transient_acts || ++since_recycle < 256 || transient_dead.empty()) return; + for (void* p : transient_dead) cactus_metal_free_shared(p); + transient_dead.clear(); + cactus_metal_session_sync(); + std::fill(metal_live.begin(), metal_live.end(), 0); + since_flush = 0; + since_recycle = 0; + }; + struct CacheWordSnap { size_t idx; size_t word; uint64_t value; }; + std::vector kv_snapshot; for (size_t i = 0; i < n; ++i) { const GraphNode& nd = *nodes_[i]; - if (nd.op_type != OpType::KV_CACHE_APPEND || nd.input_ids.size() < 2) continue; + bool kv = nd.op_type == OpType::KV_CACHE_APPEND; + bool conv = nd.op_type == OpType::CONV_CACHE_APPEND; + if ((!kv && !conv) || nd.input_ids.size() < 2) continue; auto ci = node_index_map_.find(nd.input_ids[1]); if (ci == node_index_map_.end()) continue; const uint64_t* km = reinterpret_cast(nodes_[ci->second]->output_buffer.get_data()); - if (km) kv_snapshot.push_back({ci->second, km[0]}); + if (!km) continue; + kv_snapshot.push_back({ci->second, 0, km[0]}); + if (conv) kv_snapshot.push_back({ci->second, 1, km[1]}); } auto kv_restore = [&]() { for (const auto& s : kv_snapshot) { - uint64_t* km = reinterpret_cast(nodes_[s.first]->output_buffer.get_data()); - if (km) km[0] = s.second; + uint64_t* km = reinterpret_cast(nodes_[s.idx]->output_buffer.get_data()); + if (km) km[s.word] = s.value; } }; auto input_metal_live = [&](const GraphNode& nd) -> bool { @@ -1393,6 +1509,15 @@ void CactusGraph::execute(const std::string& profile_file) { if (ap) { node->output_buffer.release_to_pool(pool); node->output_buffer.set_external(ap); + } else if (transient_acts && transient_ok[i]) { + void* tp = cactus_metal_alloc_shared(node->output_buffer.byte_size); + if (tp) { + node->output_buffer.release_to_pool(pool); + node->output_buffer.set_external(tp); + transient_ptr[i] = tp; + } else { + assign_persistent_act(*node); + } } else { assign_persistent_act(*node); } @@ -1402,11 +1527,13 @@ void CactusGraph::execute(const std::string& profile_file) { metal_live[i] = 1; for (size_t r : release_after[i]) metal_release(r); if (++since_flush >= 48) { cactus_metal_session_flush(); since_flush = 0; } + maybe_recycle(); continue; } cactus_metal_plan_disable(fplan); cactus_metal_session_sync(); kv_restore(); + release_transients(); cactus_metal_set_active(false); cactus_metal_session_end(); execute(profile_file); @@ -1432,6 +1559,7 @@ void CactusGraph::execute(const std::string& profile_file) { metal_retype_disabled_ = true; cactus_metal_session_sync(); kv_restore(); + release_transients(); cactus_metal_set_active(false); cactus_metal_session_end(); execute(profile_file); @@ -1449,7 +1577,9 @@ void CactusGraph::execute(const std::string& profile_file) { if (ot == OpType::PERSISTENT) populated_node_ids_.insert(node->id); for (size_t r : release_after[i]) metal_release(r); if (encoded && ++since_flush >= 48) { cactus_metal_session_flush(); since_flush = 0; } + if (encoded) maybe_recycle(); } + release_transients(); cactus_metal_set_active(false); cactus_metal_session_end(); cactus_graph_metal_tail_commit(); diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index e25851fc4..8bdc20895 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -958,6 +958,49 @@ MetalFusePlan* cactus_metal_plan_build( std::vector cover; if (add_cluster(c, i, cover)) plan->action[(size_t)ai] = -3; } + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd0 = *nodes[i]; + if (nd0.op_type != OpType::ADD || nd0.input_ids.size() != 2) continue; + if (nd0.output_buffer.precision != Precision::FP16 || retyped(i)) continue; + long i0 = idxof(nd0.input_ids[0]); + long i1 = idxof(nd0.input_ids[1]); + if (i0 < 0 || i1 < 0 || i0 == i1) continue; + long mm = -1, bs = -1; + if (nodes[(size_t)i0]->op_type == OpType::MATMUL) { mm = i0; bs = i1; } + else if (nodes[(size_t)i1]->op_type == OpType::MATMUL) { mm = i1; bs = i0; } + if (mm < 0 || plan->action[(size_t)mm] != -1 || pinned[(size_t)mm]) continue; + if (plan->action[(size_t)bs] == -2) continue; + if (retyped((size_t)mm) || retyped((size_t)bs)) continue; + if (cons[(size_t)mm].size() != 1) continue; + GraphNode& m = *nodes[(size_t)mm]; + if (m.input_ids.size() < 2) continue; + long li = idxof(m.input_ids[0]); + long ri = idxof(m.input_ids[1]); + if (li < 0 || ri < 0 || retyped((size_t)li) || retyped((size_t)ri)) continue; + if (plan->action[(size_t)li] == -2 || plan->action[(size_t)ri] == -2) continue; + const BufferDesc& lb = nodes[(size_t)li]->output_buffer; + const BufferDesc& rb = nodes[(size_t)ri]->output_buffer; + const BufferDesc& bb = nodes[(size_t)bs]->output_buffer; + if (lb.precision != Precision::FP16 || rb.precision != Precision::FP16 + || bb.precision != Precision::FP16) continue; + if (rb.shape.size() != 2 || lb.shape.empty()) continue; + size_t K = lb.shape.back(); + if (K == 0 || lb.total_size != K) continue; + size_t N = m.params.pretransposed_rhs ? rb.shape[0] : rb.shape[1]; + size_t rk = m.params.pretransposed_rhs ? rb.shape[1] : rb.shape[0]; + if (rk != K || N == 0 || m.output_buffer.total_size != N) continue; + if (bb.total_size != N || nd0.output_buffer.total_size != N) continue; + MetalCluster c; + c.rule = 13; + c.a0 = (size_t)li; c.a1 = (size_t)ri; c.a2 = (size_t)bs; + c.u0 = (uint32_t)K; c.u1 = (uint32_t)N; + c.b0 = m.params.pretransposed_rhs ? 1 : 0; + c.b4 = i; + std::vector cover; + cover.push_back((size_t)mm); + add_cluster(c, i, cover); + } for (size_t i = 0; i < n; ++i) { if (plan->action[i] != -1) continue; int kind, code; float p0, p1; @@ -1181,6 +1224,24 @@ void cactus_metal_plan_free(MetalFusePlan* p) { delete p; } +bool cactus_metal_plan_has_arena(const MetalFusePlan* p) { + if (!p) return false; + if (p->arena_base) return true; + for (const auto& c : p->clusters) if (c.rule == 1) return true; + return false; +} + +void cactus_metal_plan_extend_last_use(const MetalFusePlan* p, std::vector& last_use) { + if (!p) return; + const size_t n = last_use.size(); + for (const auto& c : p->clusters) { + size_t ai = c.b4; + if (ai >= n) continue; + for (size_t v : {c.a0, c.a1, c.a2, c.a3, c.a4, c.a5, c.b0, c.b1, c.b2, c.b3}) + if (v < n && last_use[v] < ai) last_use[v] = ai; + } +} + void cactus_metal_plan_disable(MetalFusePlan* p) { if (!p) return; release_cluster_buffers(p); @@ -1466,6 +1527,15 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, return cactus_metal_encode_rms_norm_add_rows(ysum, ynorm, x, res, w, c.u0, c.u1, c.f0, (int)c.b0); } + case 13: { + GraphNode& anchor = *nodes[c.b4]; + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* w = nodes[c.a1]->output_buffer.get_data(); + const void* b = nodes[c.a2]->output_buffer.get_data(); + void* y = anchor.output_buffer.get_data(); + if (!x || !w || !b || !y) return false; + return cactus_metal_encode_gemv_bias(y, x, w, b, c.u0, c.u1, (int)c.b0); + } default: return false; } } diff --git a/cactus-graph/src/ops_conv.cpp b/cactus-graph/src/ops_conv.cpp index 1033b01a0..d17f1340d 100644 --- a/cactus-graph/src/ops_conv.cpp +++ b/cactus-graph/src/ops_conv.cpp @@ -7,7 +7,6 @@ #include #include -namespace { std::vector<__fp16> dequantize_int8_weights_to_fp16( const BufferDesc& W, size_t rows, @@ -40,7 +39,6 @@ std::vector<__fp16> dequantize_int8_weights_to_fp16( } return W_fp16; } -} void compute_conv1d_causal_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { if (node.params.backend == ComputeBackend::NPU) { diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index d7a0ffce7..858fb381d 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -184,4 +184,14 @@ bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, vo uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, uint32_t sink, uint32_t W, size_t src_bytes, size_t int8_bytes, size_t scale_bytes); +bool cactus_metal_encode_conv_cache_append(void* out, const void* src, void* ring, + uint32_t hd, uint32_t ws, uint32_t nnew, uint32_t head0, uint32_t count_new, + uint32_t num_rows, int src_f32); + +bool cactus_metal_encode_rel_pos_bias(void* y, const void* q, const void* r, + uint32_t B, uint32_t T, uint32_t H, uint32_t D, uint32_t R, int r_batched, float scale); + +bool cactus_metal_encode_gemv_bias(void* out, const void* x, const void* w, const void* bias, + uint32_t K, uint32_t N, int pretransposed); + #endif diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 13c231eb0..343bdcf16 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -3060,6 +3060,70 @@ kernel void kv_append_i8_m(device const half* src [[buffer(0)]], device char* in for (uint k=0;k= N) return; + float acc = 0; + if (tr != 0u) { + device const half* wr = w + (size_t)n * K; + for (uint k = lane; k < K; k += 32u) acc = fma((float)x[k], (float)wr[k], acc); + } else { + for (uint k = lane; k < K; k += 32u) acc = fma((float)x[k], (float)w[(size_t)k * N + n], acc); + } + acc = simd_sum(acc); + if (lane == 0) y[n] = (half)((float)(half)acc + (float)bias[n]); +} +kernel void rel_pos_bias_f16(device const half* q [[buffer(0)]], device const half* r [[buffer(1)]], + device half* y [[buffer(2)]], constant uint& T [[buffer(3)]], constant uint& H [[buffer(4)]], + constant uint& D [[buffer(5)]], constant uint& rbs [[buffer(6)]], + constant float& scale [[buffer(7)]], + uint3 gid [[thread_position_in_grid]]) { + uint tj = gid.x, h = gid.y, b = gid.z; + if (tj >= T * T || h >= H) return; + uint t = tj / T, j = tj % T; + device const half* qv = q + ((size_t)b * T + t) * H * D + h * D; + uint rel = (T - 1u) - t + j; + device const half* rv = r + (size_t)b * rbs + (size_t)rel * H * D + h * D; + float acc = 0; + for (uint d = 0; d < D; ++d) acc = fma((float)qv[d], (float)rv[d], acc); + y[((size_t)b * H + h) * T * T + (size_t)t * T + j] = (half)(acc * scale); +} +kernel void conv_cache_append_f16(device const uchar* src [[buffer(0)]], + device half* ring [[buffer(1)]], device half* out [[buffer(2)]], + constant uint& hd [[buffer(3)]], constant uint& ws [[buffer(4)]], + constant uint& nnew [[buffer(5)]], constant uint& head0 [[buffer(6)]], + constant uint& count_new [[buffer(7)]], constant uint& num_rows [[buffer(8)]], + constant uint& src_f32 [[buffer(9)]], + uint2 gid [[thread_position_in_grid]]) { + uint x = gid.x, y = gid.y; + if (x >= hd || y >= ws + nnew) return; + uint start_row = num_rows - nnew; + if (y >= ws) { + uint i = y - ws; + uint sidx = (start_row + i) * hd + x; + half v = src_f32 ? (half)((device const float*)src)[sidx] : ((device const half*)src)[sidx]; + ring[((head0 + i) % ws) * hd + x] = v; + return; + } + uint pad = ws - count_new; + if (y < pad) { out[y * hd + x] = 0.0h; return; } + uint a = count_new - 1u - (y - pad); + half v; + if (a < nnew) { + uint sidx = (num_rows - 1u - a) * hd + x; + v = src_f32 ? (half)((device const float*)src)[sidx] : ((device const half*)src)[sidx]; + } else { + uint b = a - nnew; + v = ring[((head0 + 2u * ws - 1u - b) % ws) * hd + x]; + } + out[y * hd + x] = v; +} kernel void kv_append_ring_i8_m(device const half* src [[buffer(0)]], device char* int8base [[buffer(1)]], device float* scalebase [[buffer(2)]], constant uint& kv_heads [[buffer(3)]], constant uint& hdim [[buffer(4)]], constant uint& current_len [[buffer(5)]], diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 2a037ac75..815286165 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +56,7 @@ id psoBatchnorm=nil, psoGroupnorm=nil, psoBiasAddRows=nil, psoEwChain=nil; id psoAttnFlash=nil, psoRmsSimd=nil, psoScatterRows=nil; id psoTranspose2d=nil, psoBcastRows=nil, psoRmsAddSimd=nil; + id psoConvCacheAppend=nil, psoRelPosBias=nil, psoGemvBias=nil; id dummy=nil; bool ok = false; @@ -113,6 +115,8 @@ psoAttnFlash=pso("attn_flash_f16"); psoRmsSimd=pso("rms_norm_simd_f16"); psoScatterRows=pso("strided_scatter_rows_f16"); psoTranspose2d=pso("transpose2d_f16"); psoBcastRows=pso("bcast_binary_rows_f16"); psoRmsAddSimd=pso("rms_norm_add_simd_f16"); + psoConvCacheAppend=pso("conv_cache_append_f16"); psoRelPosBias=pso("rel_pos_bias_f16"); + psoGemvBias=pso("gemv_bias_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotW&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoArgmax&&psoGather; @@ -133,7 +137,20 @@ float cb_scale=1.f; id wf16=nil; }; -std::unordered_map g_resident; +std::unordered_map g_resident; + +static uint64_t resident_key(const CactusQuantMatrix* W) { + uint64_t h = 1469598103934665603ull; + auto mix = [&](uint64_t v) { h ^= v; h *= 1099511628211ull; }; + mix(W->bits); mix(W->K); mix(W->N); mix(W->group_size); mix(W->num_groups); mix(W->flags); + const uint8_t* p = (const uint8_t*)W->packed_indices; + size_t pkb = (size_t)W->N * W->num_groups * ((W->group_size * W->bits + 7u) / 8u); + if (!p || pkb == 0) return h; + size_t take = pkb < 64 ? pkb : 64; + for (size_t i = 0; i < take; ++i) mix(p[i]); + for (size_t i = pkb - take; i < pkb; ++i) mix(p[i]); + return h; +} bool g_wf16_live = false; id g_attn_scores = nil; std::unordered_map g_resident_emb; @@ -196,7 +213,8 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, ResW& resident(const CactusQuantMatrix* W) { std::lock_guard lk(g_resident_mu); - auto it = g_resident.find(W->packed_indices); + uint64_t key = resident_key(W); + auto it = g_resident.find(key); if (it != g_resident.end()) return it->second; const uint32_t K=W->K, N=W->N, ng=W->num_groups, gs=W->group_size, bits=W->bits; const uint32_t pgb=(gs*bits+7u)/8u; @@ -232,7 +250,7 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, r.packed = buf(W->packed_indices, (size_t)N*ng*pgb); r.norms = buf(W->norms, (size_t)N*ng*sizeof(__fp16)); } - return g_resident.emplace(W->packed_indices, r).first->second; + return g_resident.emplace(key, r).first->second; } id g_cmd = nil; @@ -348,7 +366,13 @@ inline void setBufAt(const void* p, size_t bytes, int idx) { void cactus_metal_set_active(bool a) { g_active = a; } bool cactus_metal_active_mode() { return ctx().ok && g_active; } -void cactus_metal_session_begin() {} +extern "C" void* objc_autoreleasePoolPush(void); +extern "C" void objc_autoreleasePoolPop(void*); +static void* g_arpool = nullptr; + +void cactus_metal_session_begin() { + if (!g_arpool) g_arpool = objc_autoreleasePoolPush(); +} static id g_last_cmd = nil; void cactus_metal_session_sync() { if (g_enc) { [g_enc endEncoding]; g_enc = nil; } @@ -366,6 +390,7 @@ void cactus_metal_session_sync() { } void cactus_metal_session_end() { cactus_metal_session_sync(); + if (g_arpool) { objc_autoreleasePoolPop(g_arpool); g_arpool = nullptr; } } void cactus_metal_invalidate_host_wraps() { std::lock_guard lk(g_resident_mu); @@ -398,6 +423,7 @@ void cactus_metal_session_flush() { static char* g_slab = nullptr; static size_t g_slab_used = 0, g_slab_cap = 0; +static std::unordered_set g_slab_bases; void* cactus_metal_alloc_pooled(size_t bytes) { if (!ctx().ok) return nullptr; if (bytes > (4u << 20)) return cactus_metal_alloc_shared(bytes); @@ -410,12 +436,14 @@ void cactus_metal_session_flush() { g_slab_used = 0; g_slab_cap = cap; g_shared[reinterpret_cast(g_slab)] = b; + g_slab_bases.insert(reinterpret_cast(g_slab)); } void* p = g_slab + g_slab_used; g_slab_used += need; return p; } void cactus_metal_free_shared(void* contents) { + if (g_slab_bases.count(reinterpret_cast(contents))) return; auto it = g_shared.find(reinterpret_cast(contents)); if (it != g_shared.end()) { g_pending.push_back(it->second); g_shared.erase(it); } } @@ -826,6 +854,8 @@ bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M) { if (!ctx().ok || !W) return false; + if (M == 1 && (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) + return cactus_metal_encode_quant_matmul(out, lhs, W); const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; const uint32_t bits=W->bits, pgb=(gs*bits+7u)/8u; @@ -1887,6 +1917,58 @@ bool cactus_metal_encode_kv_append_i8_m(const void* src, void* int8base, void* s src_bytes, int8_bytes, scale_bytes); } +bool cactus_metal_encode_gemv_bias(void* out, const void* x, const void* w, const void* bias, + uint32_t K, uint32_t N, int pretransposed) { + if (!ctx().ok || !ctx().psoGemvBias || !bias) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoGemvBias]; + setBufAt(x, (size_t)K*2, 0); + setBufAt(w, (size_t)K*N*2, 1); + setBufAt(bias, (size_t)N*2, 2); + setBufAt(out, (size_t)N*2, 3); + uint32_t trv = pretransposed ? 1u : 0u; + [g_enc setBytes:&K length:4 atIndex:4]; [g_enc setBytes:&N length:4 atIndex:5]; + [g_enc setBytes:&trv length:4 atIndex:6]; + [g_enc dispatchThreadgroups:MTLSizeMake((N+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + return true; +} + +bool cactus_metal_encode_rel_pos_bias(void* y, const void* q, const void* r, + uint32_t B, uint32_t T, uint32_t H, uint32_t D, uint32_t R, int r_batched, float scale) { + if (!ctx().ok || !ctx().psoRelPosBias) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoRelPosBias]; + setBufAt(q, (size_t)B*T*H*D*2, 0); + setBufAt(r, (size_t)(r_batched?B:1)*R*H*D*2, 1); + setBufAt(y, (size_t)B*H*T*T*2, 2); + uint32_t rbs = r_batched ? R*H*D : 0u; + [g_enc setBytes:&T length:4 atIndex:3]; [g_enc setBytes:&H length:4 atIndex:4]; + [g_enc setBytes:&D length:4 atIndex:5]; [g_enc setBytes:&rbs length:4 atIndex:6]; + [g_enc setBytes:&scale length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake((size_t)T*T, H, B) + threadsPerThreadgroup:MTLSizeMake(64, H < 4 ? H : 4, 1)]; + return true; +} + +bool cactus_metal_encode_conv_cache_append(void* out, const void* src, void* ring, + uint32_t hd, uint32_t ws, uint32_t nnew, uint32_t head0, uint32_t count_new, + uint32_t num_rows, int src_f32) { + if (!ctx().ok || !ctx().psoConvCacheAppend) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoConvCacheAppend]; + setBufAt(src, (size_t)num_rows*hd*(src_f32?4:2), 0); + setBufAt(ring, (size_t)ws*hd*2, 1); + setBufAt(out, (size_t)ws*hd*2, 2); + uint32_t f32 = src_f32 ? 1u : 0u; + [g_enc setBytes:&hd length:4 atIndex:3]; [g_enc setBytes:&ws length:4 atIndex:4]; + [g_enc setBytes:&nnew length:4 atIndex:5]; [g_enc setBytes:&head0 length:4 atIndex:6]; + [g_enc setBytes:&count_new length:4 atIndex:7]; [g_enc setBytes:&num_rows length:4 atIndex:8]; + [g_enc setBytes:&f32 length:4 atIndex:9]; + [g_enc dispatchThreads:MTLSizeMake(hd, ws + nnew, 1) + threadsPerThreadgroup:MTLSizeMake(hd < 64 ? hd : 64, 4, 1)]; + return true; +} + bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, void* scalebase, uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, uint32_t sink, uint32_t W, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index ea1713f5a..fb5efdeb3 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -106,4 +106,10 @@ bool cactus_metal_encode_kv_append_i8_m(const void*, void*, void*, uint32_t, uin uint32_t, uint32_t, size_t, size_t, size_t) { return false; } bool cactus_metal_encode_kv_append_ring_i8_m(const void*, void*, void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_conv_cache_append(void*, const void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_rel_pos_bias(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, int, float) { return false; } +bool cactus_metal_encode_gemv_bias(void*, const void*, const void*, const void*, + uint32_t, uint32_t, int) { return false; } #endif From 9a077bc3de70a904637693e129ca1b396c45a9b1 Mon Sep 17 00:00:00 2001 From: jakmro Date: Fri, 3 Jul 2026 14:01:37 +0200 Subject: [PATCH 25/36] c Signed-off-by: jakmro --- cactus-graph/src/execute.cpp | 10 ++- cactus-graph/src/metal_plan.cpp | 23 +++--- cactus-kernels/src/cactus_kernels.metal | 93 ++++++++++--------------- cactus-kernels/src/metal_backend.mm | 44 +++++++----- 4 files changed, 87 insertions(+), 83 deletions(-) diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index cd29e22bb..8b34709b4 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -1368,13 +1368,13 @@ void CactusGraph::execute(const std::string& profile_file) { } std::vector keep_until_graph_cleanup(n, false); - for (size_t i = 0; i < n; ++i) { + for (size_t i = n; i-- > 0;) { const auto& node = *nodes_[i]; if (!aliases_input(node) || node.input_ids.empty()) continue; auto it = node_index_map_.find(node.input_ids[0]); if (it == node_index_map_.end()) continue; size_t base_idx = it->second; - if (use_count[i] == 0) { + if (use_count[i] == 0 || keep_until_graph_cleanup[i]) { keep_until_graph_cleanup[base_idx] = true; } else { last_use[base_idx] = std::max(last_use[base_idx], last_use[i]); @@ -1474,6 +1474,12 @@ void CactusGraph::execute(const std::string& profile_file) { }; const std::vector* elist = fplan ? cactus_metal_plan_exec_list(fplan) : nullptr; size_t iter_count = elist ? elist->size() : n; + if (fplan) { + for (size_t ii = 0; ii < iter_count; ++ii) { + size_t i = elist ? (size_t)(*elist)[ii] : ii; + if (cactus_metal_plan_action(fplan, i) == -3) assign_persistent_act(*nodes_[i]); + } + } for (size_t ii = 0; ii < iter_count; ++ii) { size_t i = elist ? (size_t)(*elist)[ii] : ii; auto& node = nodes_[i]; diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index 8bdc20895..f17ec19c7 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -215,8 +215,13 @@ MetalFusePlan* cactus_metal_plan_build( return i; }; + auto release_scratch = [&](MetalCluster& c) { + if (c.s0) { cactus_metal_free_shared(c.s0); c.s0 = nullptr; } + if (c.s1) { cactus_metal_free_shared(c.s1); c.s1 = nullptr; } + for (auto& sp : c.sc) if (sp) { cactus_metal_free_shared(sp); sp = nullptr; } + }; auto add_cluster = [&](MetalCluster c, size_t anchor, const std::vector& cover) -> bool { - for (size_t v : cover) if (v != anchor && pinned[v]) return false; + for (size_t v : cover) if (v != anchor && pinned[v]) { release_scratch(c); return false; } int32_t cid = (int32_t)plan->clusters.size(); plan->clusters.push_back(c); for (size_t v : cover) if (v != anchor) plan->action[v] = -2; @@ -792,12 +797,7 @@ MetalFusePlan* cactus_metal_plan_build( for (size_t base = 0; base + 1 < mms.size(); base += 3) { size_t cnt = std::min(3, mms.size() - base); if (cnt < 2) break; - size_t anchor_i = mms[base + cnt - 1]; - bool ordered = true; - for (size_t mi = 0; mi < cnt && ordered; ++mi) - for (size_t cc : cons[mms[base + mi]]) - if (cc < anchor_i && plan->action[cc] != -2) { ordered = false; break; } - if (!ordered) continue; + size_t anchor_i = mms[base]; MetalCluster c; c.rule = 2; c.a0 = (size_t)src; @@ -805,6 +805,7 @@ MetalFusePlan* cactus_metal_plan_build( c.b1 = mms[base + 1]; c.b2 = cnt > 2 ? mms[base + 2] : 0; c.u0 = (uint32_t)cnt; + c.b4 = anchor_i; size_t maxK = 0; for (size_t mi = 0; mi < cnt; ++mi) { const BufferDesc& rb = nodes[(size_t)idxof(nodes[mms[base + mi]]->input_ids[1])]->output_buffer; @@ -814,8 +815,8 @@ MetalFusePlan* cactus_metal_plan_build( for (uint32_t bi = 0; bi < c.u0; ++bi) if (!c.sc[bi]) c.sc[bi] = cactus_metal_alloc_shared(maxK * 2); std::vector cover; - if (add_cluster(c, mms[base + cnt - 1], cover)) - for (size_t mi = 0; mi + 1 < cnt; ++mi) plan->action[mms[base + mi]] = -3; + if (add_cluster(c, anchor_i, cover)) + for (size_t mi = 1; mi < cnt; ++mi) plan->action[mms[base + mi]] = -3; } } } @@ -1177,6 +1178,7 @@ MetalFusePlan* cactus_metal_plan_build( size_t i = ui; const GraphNode& nd = *nodes[i]; if (plan->action[i] == -2) continue; + if (pinned[i]) continue; if (nd.op_type == OpType::KV_CACHE_STATE || nd.op_type == OpType::CONV_CACHE_STATE || nd.op_type == OpType::RECURRENT_CACHE_STATE || nd.op_type == OpType::KV_CACHE_APPEND || nd.op_type == OpType::PERSISTENT) continue; @@ -1256,6 +1258,7 @@ int32_t cactus_metal_plan_action(const MetalFusePlan* p, size_t i) { return p && i < p->action.size() ? p->action[i] : -1; } + const std::vector* cactus_metal_plan_exec_list(const MetalFusePlan* p) { return p ? &p->exec_list : nullptr; } @@ -1291,6 +1294,7 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, if (!km || !vm) return false; size_t clen = km[0], mx = km[1], sink = km[4]; uint32_t hd = c.u1 & 0xFFFFu, nqh = c.u0; + if (km[2] != 1 || vm[3] != km[3] || hd == 0 || hd > 512u) return false; bool has_new = (c.u1 & 0x10000u) != 0; size_t ng = (hd + 31) / 32; size_t win = anchor.params.window_size; @@ -1337,6 +1341,7 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, if (!outs[bi] || !c.sc[bi]) return false; } const void* x = nodes[c.a0]->output_buffer.get_data(); + if (!x) return false; void* codes[3] = { c.sc[0], c.sc[1], c.sc[2] }; if (cactus_metal_encode_transform_batch(x, Wp, (int)c.u0, codes)) { const void* ccodes[3] = { c.sc[0], c.sc[1], c.sc[2] }; diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 343bdcf16..6f4395f18 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -17,6 +17,28 @@ inline float gelu_tanh(float x) { return 0.5f * x * (1.0f + precise::tanh(c)); } +static inline void cq4_hada128(thread float& x0, thread float& x1, + thread float& x2, thread float& x3, uint lane) { + float a0=x0+x1,a1=x0-x1,a2=x2+x3,a3=x2-x3; + x0=a0+a2; x1=a1+a3; x2=a0-a2; x3=a1-a3; + #pragma clang loop unroll(full) + for (uint d=1u; d<=16u; d<<=1){ + bool hi=(lane&d)!=0u; + float p0=simd_shuffle_xor(x0,d),p1=simd_shuffle_xor(x1,d),p2=simd_shuffle_xor(x2,d),p3=simd_shuffle_xor(x3,d); + x0=hi?p0-x0:x0+p0; x1=hi?p1-x1:x1+p1; x2=hi?p2-x2:x2+p2; x3=hi?p3-x3:x3+p3; + } + float s=rsqrt(128.0f); + x0*=s; x1*=s; x2*=s; x3*=s; +} + +static inline float cq4_dot64(half4 c0, half4 c1, half4 c2, half4 c3, + ushort4 w, threadgroup const float* cb) { + return (float)c0.x*cb[w[0]&0xF] + (float)c0.y*cb[(w[0]>>4)&0xF] + (float)c0.z*cb[(w[0]>>8)&0xF] + (float)c0.w*cb[(w[0]>>12)&0xF] + + (float)c1.x*cb[w[1]&0xF] + (float)c1.y*cb[(w[1]>>4)&0xF] + (float)c1.z*cb[(w[1]>>8)&0xF] + (float)c1.w*cb[(w[1]>>12)&0xF] + + (float)c2.x*cb[w[2]&0xF] + (float)c2.y*cb[(w[2]>>4)&0xF] + (float)c2.z*cb[(w[2]>>8)&0xF] + (float)c2.w*cb[(w[2]>>12)&0xF] + + (float)c3.x*cb[w[3]&0xF] + (float)c3.y*cb[(w[3]>>4)&0xF] + (float)c3.z*cb[(w[3]>>8)&0xF] + (float)c3.w*cb[(w[3]>>12)&0xF]; +} + kernel void cq4_transform( device const half* x [[buffer(0)]], device const half* recip [[buffer(1)]], @@ -60,17 +82,9 @@ kernel void cq4_transform_simd( float x1=(float)x[b+1]*(float)recip[b+1]*(float)lsign[k+1]; float x2=(float)x[b+2]*(float)recip[b+2]*(float)lsign[k+2]; float x3=(float)x[b+3]*(float)recip[b+3]*(float)lsign[k+3]; - float a0=x0+x1,a1=x0-x1,a2=x2+x3,a3=x2-x3; - x0=a0+a2; x1=a1+a3; x2=a0-a2; x3=a1-a3; - #pragma clang loop unroll(full) - for (uint d=1u; d<=16u; d<<=1){ - bool hi=(lane&d)!=0u; - float p0=simd_shuffle_xor(x0,d),p1=simd_shuffle_xor(x1,d),p2=simd_shuffle_xor(x2,d),p3=simd_shuffle_xor(x3,d); - x0=hi?p0-x0:x0+p0; x1=hi?p1-x1:x1+p1; x2=hi?p2-x2:x2+p2; x3=hi?p3-x3:x3+p3; - } - float s=rsqrt(128.0f); - zmem[k+0]=x0*s*(float)rsign[k+0]; zmem[k+1]=x1*s*(float)rsign[k+1]; - zmem[k+2]=x2*s*(float)rsign[k+2]; zmem[k+3]=x3*s*(float)rsign[k+3]; + cq4_hada128(x0, x1, x2, x3, lane); + zmem[k+0]=x0*(float)rsign[k+0]; zmem[k+1]=x1*(float)rsign[k+1]; + zmem[k+2]=x2*(float)rsign[k+2]; zmem[k+3]=x3*(float)rsign[k+3]; threadgroup_barrier(mem_flags::mem_threadgroup); code[b+0]=(half)zmem[perm[k+0]]; code[b+1]=(half)zmem[perm[k+1]]; code[b+2]=(half)zmem[perm[k+2]]; code[b+3]=(half)zmem[perm[k+3]]; @@ -98,17 +112,9 @@ kernel void cq4_transform_batch( float x1=(float)x[base+1]*(float)recip[base+1]*(float)lsign[k+1]; float x2=(float)x[base+2]*(float)recip[base+2]*(float)lsign[k+2]; float x3=(float)x[base+3]*(float)recip[base+3]*(float)lsign[k+3]; - float a0=x0+x1,a1=x0-x1,a2=x2+x3,a3=x2-x3; - x0=a0+a2; x1=a1+a3; x2=a0-a2; x3=a1-a3; - #pragma clang loop unroll(full) - for (uint d=1u; d<=16u; d<<=1){ - bool hi=(lane&d)!=0u; - float p0=simd_shuffle_xor(x0,d),p1=simd_shuffle_xor(x1,d),p2=simd_shuffle_xor(x2,d),p3=simd_shuffle_xor(x3,d); - x0=hi?p0-x0:x0+p0; x1=hi?p1-x1:x1+p1; x2=hi?p2-x2:x2+p2; x3=hi?p3-x3:x3+p3; - } - float s=rsqrt(128.0f); - zmem[k+0]=x0*s*(float)rsign[k+0]; zmem[k+1]=x1*s*(float)rsign[k+1]; - zmem[k+2]=x2*s*(float)rsign[k+2]; zmem[k+3]=x3*s*(float)rsign[k+3]; + cq4_hada128(x0, x1, x2, x3, lane); + zmem[k+0]=x0*(float)rsign[k+0]; zmem[k+1]=x1*(float)rsign[k+1]; + zmem[k+2]=x2*(float)rsign[k+2]; zmem[k+3]=x3*(float)rsign[k+3]; threadgroup_barrier(mem_flags::mem_threadgroup); code[base+0]=(half)zmem[perm[k+0]]; code[base+1]=(half)zmem[perm[k+1]]; code[base+2]=(half)zmem[perm[k+2]]; code[base+3]=(half)zmem[perm[k+3]]; @@ -191,10 +197,7 @@ kernel void cq4_gemv_mr( uint n = n0+r; device const ushort4* pr = (device const ushort4*)(packed + ((size_t)n*num_groups+g)*pgb + off/2u); ushort4 w = pr[0]; - float p = (float)c0.x*cb[w[0]&0xF] + (float)c0.y*cb[(w[0]>>4)&0xF] + (float)c0.z*cb[(w[0]>>8)&0xF] + (float)c0.w*cb[(w[0]>>12)&0xF] - + (float)c1.x*cb[w[1]&0xF] + (float)c1.y*cb[(w[1]>>4)&0xF] + (float)c1.z*cb[(w[1]>>8)&0xF] + (float)c1.w*cb[(w[1]>>12)&0xF] - + (float)c2.x*cb[w[2]&0xF] + (float)c2.y*cb[(w[2]>>4)&0xF] + (float)c2.z*cb[(w[2]>>8)&0xF] + (float)c2.w*cb[(w[2]>>12)&0xF] - + (float)c3.x*cb[w[3]&0xF] + (float)c3.y*cb[(w[3]>>4)&0xF] + (float)c3.z*cb[(w[3]>>8)&0xF] + (float)c3.w*cb[(w[3]>>12)&0xF]; + float p = cq4_dot64(c0, c1, c2, c3, w, cb); acc[r] += (float)norms[(size_t)n*num_groups+g]*p; } } @@ -238,17 +241,9 @@ kernel void cq4_transform_gemv( float x1=(float)x[base+1]*(float)recip[base+1]*(float)lsign[k+1]; float x2=(float)x[base+2]*(float)recip[base+2]*(float)lsign[k+2]; float x3=(float)x[base+3]*(float)recip[base+3]*(float)lsign[k+3]; - float a0=x0+x1,a1=x0-x1,a2=x2+x3,a3=x2-x3; - x0=a0+a2; x1=a1+a3; x2=a0-a2; x3=a1-a3; - #pragma clang loop unroll(full) - for (uint d=1u; d<=16u; d<<=1){ - bool hi=(lane&d)!=0u; - float p0=simd_shuffle_xor(x0,d),p1=simd_shuffle_xor(x1,d),p2=simd_shuffle_xor(x2,d),p3=simd_shuffle_xor(x3,d); - x0=hi?p0-x0:x0+p0; x1=hi?p1-x1:x1+p1; x2=hi?p2-x2:x2+p2; x3=hi?p3-x3:x3+p3; - } - float sc=rsqrt(128.0f); - zwave[sg*128u+k+0]=x0*sc*(float)rsign[k+0]; zwave[sg*128u+k+1]=x1*sc*(float)rsign[k+1]; - zwave[sg*128u+k+2]=x2*sc*(float)rsign[k+2]; zwave[sg*128u+k+3]=x3*sc*(float)rsign[k+3]; + cq4_hada128(x0, x1, x2, x3, lane); + zwave[sg*128u+k+0]=x0*(float)rsign[k+0]; zwave[sg*128u+k+1]=x1*(float)rsign[k+1]; + zwave[sg*128u+k+2]=x2*(float)rsign[k+2]; zwave[sg*128u+k+3]=x3*(float)rsign[k+3]; } threadgroup_barrier(mem_flags::mem_threadgroup); if (g < ng) { @@ -270,10 +265,7 @@ kernel void cq4_transform_gemv( uint n = n0+r; device const ushort4* pr = (device const ushort4*)(packed + ((size_t)n*ng+g)*64u + off/2u); ushort4 w = pr[0]; - float p = (float)c0.x*cb[w[0]&0xF] + (float)c0.y*cb[(w[0]>>4)&0xF] + (float)c0.z*cb[(w[0]>>8)&0xF] + (float)c0.w*cb[(w[0]>>12)&0xF] - + (float)c1.x*cb[w[1]&0xF] + (float)c1.y*cb[(w[1]>>4)&0xF] + (float)c1.z*cb[(w[1]>>8)&0xF] + (float)c1.w*cb[(w[1]>>12)&0xF] - + (float)c2.x*cb[w[2]&0xF] + (float)c2.y*cb[(w[2]>>4)&0xF] + (float)c2.z*cb[(w[2]>>8)&0xF] + (float)c2.w*cb[(w[2]>>12)&0xF] - + (float)c3.x*cb[w[3]&0xF] + (float)c3.y*cb[(w[3]>>4)&0xF] + (float)c3.z*cb[(w[3]>>8)&0xF] + (float)c3.w*cb[(w[3]>>12)&0xF]; + float p = cq4_dot64(c0, c1, c2, c3, w, cb); acc[r] += (float)norms[(size_t)n*ng+g]*p; } } @@ -318,17 +310,9 @@ kernel void cq4_swiglu_transform( x2=(float)(half)clamp(g2,-65504.0f,65504.0f)*(float)recip[b+2]*(float)lsign[k+2]; x3=(float)(half)clamp(g3,-65504.0f,65504.0f)*(float)recip[b+3]*(float)lsign[k+3]; } - float a0=x0+x1,a1=x0-x1,a2=x2+x3,a3=x2-x3; - x0=a0+a2; x1=a1+a3; x2=a0-a2; x3=a1-a3; - #pragma clang loop unroll(full) - for (uint d=1u; d<=16u; d<<=1){ - bool hi=(lane&d)!=0u; - float p0=simd_shuffle_xor(x0,d),p1=simd_shuffle_xor(x1,d),p2=simd_shuffle_xor(x2,d),p3=simd_shuffle_xor(x3,d); - x0=hi?p0-x0:x0+p0; x1=hi?p1-x1:x1+p1; x2=hi?p2-x2:x2+p2; x3=hi?p3-x3:x3+p3; - } - float s=rsqrt(128.0f); - zmem[k+0]=x0*s*(float)rsign[k+0]; zmem[k+1]=x1*s*(float)rsign[k+1]; - zmem[k+2]=x2*s*(float)rsign[k+2]; zmem[k+3]=x3*s*(float)rsign[k+3]; + cq4_hada128(x0, x1, x2, x3, lane); + zmem[k+0]=x0*(float)rsign[k+0]; zmem[k+1]=x1*(float)rsign[k+1]; + zmem[k+2]=x2*(float)rsign[k+2]; zmem[k+3]=x3*(float)rsign[k+3]; threadgroup_barrier(mem_flags::mem_threadgroup); code[b+0]=(half)zmem[perm[k+0]]; code[b+1]=(half)zmem[perm[k+1]]; code[b+2]=(half)zmem[perm[k+2]]; code[b+3]=(half)zmem[perm[k+3]]; @@ -373,10 +357,7 @@ kernel void cq4_gemv_cat( uint n = nl0+r; device const ushort4* pr = (device const ushort4*)(packed + ((size_t)n*ng+g)*64u + off/2u); ushort4 w = pr[0]; - float p = (float)c0.x*cb[w[0]&0xF] + (float)c0.y*cb[(w[0]>>4)&0xF] + (float)c0.z*cb[(w[0]>>8)&0xF] + (float)c0.w*cb[(w[0]>>12)&0xF] - + (float)c1.x*cb[w[1]&0xF] + (float)c1.y*cb[(w[1]>>4)&0xF] + (float)c1.z*cb[(w[1]>>8)&0xF] + (float)c1.w*cb[(w[1]>>12)&0xF] - + (float)c2.x*cb[w[2]&0xF] + (float)c2.y*cb[(w[2]>>4)&0xF] + (float)c2.z*cb[(w[2]>>8)&0xF] + (float)c2.w*cb[(w[2]>>12)&0xF] - + (float)c3.x*cb[w[3]&0xF] + (float)c3.y*cb[(w[3]>>4)&0xF] + (float)c3.z*cb[(w[3]>>8)&0xF] + (float)c3.w*cb[(w[3]>>12)&0xF]; + float p = cq4_dot64(c0, c1, c2, c3, w, cb); acc[r] += (float)norms[(size_t)n*ng+g]*p; } } diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 815286165..e41bb6d5a 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -136,6 +136,7 @@ id cb_i8=nil; float cb_scale=1.f; id wf16=nil; + bool wf16_volatile=false; }; std::unordered_map g_resident; @@ -383,6 +384,9 @@ void cactus_metal_session_sync() { } if (g_last_cmd) { [g_last_cmd waitUntilCompleted]; + if (g_last_cmd.status == MTLCommandBufferStatusError) + fprintf(stderr, "[cactus-metal] command buffer failed: %s\n", + g_last_cmd.error ? [[g_last_cmd.error localizedDescription] UTF8String] : "unknown"); g_last_cmd = nil; } for (id b : g_pending) g_free[(size_t)b.length].push_back(b); @@ -667,7 +671,12 @@ static inline bool quant_lowbit_eligible(const CactusQuantMatrix* W) { void cactus_metal_trim_prefill_cache() { if (!g_wf16_live) return; std::lock_guard lk(g_resident_mu); - for (auto& kv : g_resident) kv.second.wf16 = nil; + for (auto& kv : g_resident) { + if (kv.second.wf16 && !kv.second.wf16_volatile) { + [kv.second.wf16 setPurgeableState:MTLPurgeableStateVolatile]; + kv.second.wf16_volatile = true; + } + } g_wf16_live = false; } @@ -878,9 +887,16 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; if ((g_predequant && ctx().psoGdense && ctx().psoDeq) || lowbit) { + bool need_dequant = false; if (!rw.wf16) { rw.wf16 = [ctx().dev newBufferWithLength:(size_t)N*K*2 options:MTLResourceStorageModeShared]; - g_wf16_live = true; + need_dequant = true; + } else if (rw.wf16_volatile) { + need_dequant = [rw.wf16 setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty; + } + rw.wf16_volatile = false; + g_wf16_live = true; + if (need_dequant) { [g_enc setComputePipelineState:(lowbit?ctx().psoDeqLow:ctx().psoDeq)]; [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.wf16 offset:0 atIndex:3]; @@ -1095,6 +1111,11 @@ bool cactus_metal_encode_gather_f16(void* out, const void* table, size_t table_b return true; } +static void setCacheAt(const void* p, size_t bytes, int idx) { + if (p && bytes) setBufAt(p, bytes, idx); + else [g_enc setBuffer:ctx().dummy offset:0 atIndex:idx]; +} + bool cactus_metal_encode_attention_i8( void* out, const void* q, const void* knew, const void* vnew, const void* kc, const void* vc, const void* ks, const void* vs, @@ -1105,10 +1126,6 @@ bool cactus_metal_encode_attention_i8( if (kv_end <= kv_start || (num_q_heads % num_kv_heads) != 0) return false; if (head_dim > 512u || v_hdim > 512u) return false; ensureEncoder(); - auto setCache = [&](const void* p, size_t bytes, int idx) { - if (p && bytes) setBufAt(p, bytes, idx); - else [g_enc setBuffer:ctx().dummy offset:0 atIndex:idx]; - }; auto setInputs = [&]() { setBufAt(q, (size_t)num_q_heads*head_dim*2, 0); if (total_keys > history_len && knew && vnew) { @@ -1117,8 +1134,8 @@ bool cactus_metal_encode_attention_i8( } else { [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; } - setCache(kc, kc_bytes, 3); setCache(vc, vc_bytes, 4); - setCache(ks, ks_bytes, 5); setCache(vs, vs_bytes, 6); + setCacheAt(kc, kc_bytes, 3); setCacheAt(vc, vc_bytes, 4); + setCacheAt(ks, ks_bytes, 5); setCacheAt(vs, vs_bytes, 6); }; const uint32_t T = 256, nsg = T / 32u; @@ -1209,6 +1226,7 @@ bool cactus_metal_encode_attention_i8_prefill( const uint32_t T = 128; if (total_keys == 0 || M == 0 || (num_q_heads % num_kv_heads) != 0) return false; uint32_t maxsc = (ring > 0u) ? ((total_keys > sink + ring) ? (sink + ring) : total_keys) : 256u; + if (maxsc > 7936u) return false; bool mma2 = (ring == 0u && window == 0u && is_causal && head_dim == 512u && v_hdim == 512u && num_q_heads == 8u && num_kv_heads == 1u); bool hd256 = (ctx().psoAttnPreHd256 != nil && is_causal && head_dim == 256u && v_hdim == 256u @@ -1223,10 +1241,7 @@ bool cactus_metal_encode_attention_i8_prefill( } else { [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; } - auto setCacheM = [&](const void* p, size_t bytes, int idx) { - if (p && bytes) setBufAt(p, bytes, idx); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:idx]; - }; - setCacheM(kc, kc_bytes, 3); setCacheM(vc, vc_bytes, 4); setCacheM(ks, ks_bytes, 5); setCacheM(vs, vs_bytes, 6); + setCacheAt(kc, kc_bytes, 3); setCacheAt(vc, vc_bytes, 4); setCacheAt(ks, ks_bytes, 5); setCacheAt(vs, vs_bytes, 6); setBufAt(out, (size_t)M*num_q_heads*v_hdim*2, 7); [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; [g_enc setBytes:&head_dim length:4 atIndex:10]; [g_enc setBytes:&v_hdim length:4 atIndex:11]; @@ -1247,10 +1262,7 @@ bool cactus_metal_encode_attention_i8_prefill( } else { [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; } - auto setCache = [&](const void* p, size_t bytes, int idx) { - if (p && bytes) setBufAt(p, bytes, idx); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:idx]; - }; - setCache(kc, kc_bytes, 3); setCache(vc, vc_bytes, 4); setCache(ks, ks_bytes, 5); setCache(vs, vs_bytes, 6); + setCacheAt(kc, kc_bytes, 3); setCacheAt(vc, vc_bytes, 4); setCacheAt(ks, ks_bytes, 5); setCacheAt(vs, vs_bytes, 6); setBufAt(out, (size_t)M*num_q_heads*v_hdim*2, 7); [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; [g_enc setBytes:&head_dim length:4 atIndex:10]; [g_enc setBytes:&v_hdim length:4 atIndex:11]; From ac1af8c5abdd9d42606b71224c21aba84fb46e39 Mon Sep 17 00:00:00 2001 From: jakmro Date: Sat, 4 Jul 2026 15:45:05 +0200 Subject: [PATCH 26/36] fixes --- cactus-engine/src/complete.cpp | 3 + cactus-engine/src/engine.h | 7 +- cactus-engine/src/model.cpp | 8 +- cactus-engine/src/utils.h | 8 +- cactus-graph/src/execute.cpp | 88 ++++++--- cactus-graph/src/metal_plan.cpp | 17 +- cactus-graph/src/ops_cache.cpp | 11 +- cactus-graph/tests/test_metal_parity.cpp | 214 ++++++++++++++++++++++ cactus-kernels/metal_backend.h | 1 + cactus-kernels/src/cactus_kernels.metal | 74 +++++--- cactus-kernels/src/metal_backend.mm | 125 +++++++++++-- cactus-kernels/src/metal_backend_stub.cpp | 1 + 12 files changed, 466 insertions(+), 91 deletions(-) diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index c376a85ed..0821cf9bb 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -829,6 +829,9 @@ int cactus_complete( } auto* tokenizer = handle->model->get_tokenizer(); auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, true, true, pcm_buffer, pcm_buffer_size); + if (prompt.options.sample_seed != 0) { + handle->model->set_sample_seed(prompt.options.sample_seed); + } CACTUS_LOG_DEBUG("complete", "Prompt tokens: " << prompt.tokens.size() << ", max_tokens: " << prompt.options.max_tokens); diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index dfe059329..220f3595a 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -637,7 +637,8 @@ class Model { uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f); + float min_p = 0.15f, float repetition_penalty = 1.0f); + void set_sample_seed(uint64_t seed) { sample_rng_.seed(static_cast(seed)); } bool prefill_and_sample_first_token(const std::vector& tokens, uint32_t& out_token, float* out_uncertainty = nullptr); @@ -668,13 +669,13 @@ class Model { uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f); + float min_p = 0.15f, float repetition_penalty = 1.0f); uint32_t decode_with_audio(const std::vector& tokens, const std::vector>& audio_features_per_message, float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, + float min_p = 0.15f, float repetition_penalty = 1.0f, float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); struct ParakeetTdtStreamState { diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 9628b08b3..c8a5cb6c9 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -3213,6 +3213,10 @@ uint32_t Model::decode(const std::vector& tokens, float temperature, f size_t tk = top_k == 0 ? config_.default_top_k : top_k; const bool greedy = temp <= 0.011f; prepare_sampling_context(repetition_penalty); + struct SampClearGuard { + Model* m; + ~SampClearGuard() { cactus_graph_clear_sampling(); m->samp_ctx_active_ = false; } + } samp_guard{this}; if (decode_route_ == DecodeRoute::ENCODER_CROSS_KV_STEP) { if (!encoder_cross_kv_ready_ && encoder_cross_kv_source_kind_ == "text_tokens") { std::vector source_tokens = tokens; @@ -3247,8 +3251,6 @@ uint32_t Model::decode(const std::vector& tokens, float temperature, f uint32_t result = (greedy && !samp_ctx_active_) ? argmax_last_logits(out_entropy) : sample_component_logits(*decoder_, temp, tp, tk, min_p, greedy, out_entropy); - cactus_graph_clear_sampling(); - samp_ctx_active_ = false; record_sampled_token(result); return result; } @@ -3262,8 +3264,6 @@ uint32_t Model::decode(const std::vector& tokens, float temperature, f uint32_t result = (greedy && !samp_ctx_active_) ? argmax_last_logits(out_entropy) : sample_component_logits(*decoder_, temp, tp, tk, min_p, greedy, out_entropy); - cactus_graph_clear_sampling(); - samp_ctx_active_ = false; record_sampled_token(result); return result; } diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 23f6b399c..b0aaed8ea 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -507,7 +507,8 @@ struct InferenceOptions { float temperature = 0.0f; float top_p = 0.0f; float min_p = 0.15f; - float repetition_penalty = 1.1f; + float repetition_penalty = 1.0f; + uint64_t sample_seed = 0; float confidence_threshold = -1.0f; size_t top_k = 0; size_t max_tokens = 100; @@ -1619,6 +1620,11 @@ inline InferenceOptions parse_inference_options_json(const std::string& json) { } } + size_t parsed_seed = 0; + if (try_parse_json_uint(json, "seed", parsed_seed)) { + options.sample_seed = parsed_seed; + } + size_t parsed_top_k = options.top_k; if (try_parse_json_uint(json, "top_k", parsed_top_k)) { options.top_k = parsed_top_k; diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 8b34709b4..f2125f6af 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -979,6 +979,18 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n max_seq*kv_heads*ngK*sizeof(float), v_max*kv_heads*ngV*sizeof(float), (uint32_t)sink, ringv); } + size_t cache_ceiling = nodes[map.at(node.input_ids[3])]->params.max_cache_seq_len; + if (win > 0 && win < cache_ceiling) { + size_t sink = km[4]; + size_t ringW = max_seq > sink + 1 ? max_seq - sink - 1 : 0; + if (ringW == 0) return false; + if (total_keys > ringW) { + history_len = ringW; + total_keys = ringW; + kv_start = 0; + kv_end = ringW; + } + } bool ok = cactus_metal_encode_attention_i8( out.get_data(), qb.get_data(), knew.get_data(), vnew.get_data(), bk + 64, bv + 64, @@ -1006,6 +1018,7 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n size_t ngK = (hdim + 31)/32; if (new_seq_len > 1) { if (sliding) { + if (max_len <= sink + 1) return false; uint32_t W = (uint32_t)(max_len - sink - 1); if (!cactus_metal_encode_kv_append_ring_i8_m(new_kv.get_data(), base + 64, base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, @@ -1043,13 +1056,25 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n if (out.get_data()) *out.data_as() = static_cast(km[0]); return true; } - if (!sliding && new_total > max_len) { + if (sliding) { + if (max_len <= sink + 1) return false; + uint32_t W = (uint32_t)(max_len - sink - 1); + if (!cactus_metal_encode_kv_append_ring_i8_m(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)current_len, 32, 1u, (uint32_t)sink, W, + new_kv.byte_size, max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = new_total; + if (out.get_data()) *out.data_as() = static_cast(new_total); + return true; + } + if (new_total > max_len) { if (!cactus_kv_cache_grow(cache, new_total, ceiling)) return false; km = reinterpret_cast(cache.get_data()); max_len = km[1]; base = static_cast(cache.get_data()); if (new_total > max_len) return false; } - size_t window = sliding ? ws : max_len; + size_t window = max_len; if (new_total > window) { size_t keep_sink = std::min({sink, current_len, window}); size_t tail_capacity = window - keep_sink; @@ -1390,12 +1415,38 @@ void CactusGraph::execute(const std::string& profile_file) { } if (metal_mode && !need_debug) { - cactus_metal_session_begin(); - cactus_metal_set_active(true); const bool transient_acts = n >= 1500 && !cactus_metal_plan_has_arena(fplan); std::vector transient_ptr(transient_acts ? n : 0, nullptr); std::vector transient_dead; std::vector transient_ok(transient_acts ? n : 0, 0); + auto release_transients = [&]() { + for (void* p : transient_dead) cactus_metal_free_shared(p); + transient_dead.clear(); + for (size_t i = 0; i < transient_ptr.size(); ++i) + if (transient_ptr[i]) { cactus_metal_free_shared(transient_ptr[i]); transient_ptr[i] = nullptr; } + }; + struct CacheWordSnap { size_t idx; size_t word; uint64_t value; }; + std::vector kv_snapshot; + auto kv_restore = [&]() { + for (const auto& s : kv_snapshot) { + uint64_t* km = reinterpret_cast(nodes_[s.idx]->output_buffer.get_data()); + if (km) km[s.word] = s.value; + } + }; + auto metal_abort_cleanup = [&]() { + cactus_metal_session_sync(); + kv_restore(); + release_transients(); + cactus_metal_set_active(false); + cactus_metal_session_end(); + }; + struct MetalExecGuard { + decltype(metal_abort_cleanup)& cleanup; + bool armed = true; + ~MetalExecGuard() { if (armed) cleanup(); } + } metal_guard{metal_abort_cleanup}; + cactus_metal_session_begin(); + cactus_metal_set_active(true); if (transient_acts) { for (size_t i = 0; i < n; ++i) { size_t id = nodes_[i]->id; @@ -1405,12 +1456,6 @@ void CactusGraph::execute(const std::string& profile_file) { && nodes_[i]->output_buffer.byte_size >= (256u << 10); } } - auto release_transients = [&]() { - for (void* p : transient_dead) cactus_metal_free_shared(p); - transient_dead.clear(); - for (size_t i = 0; i < transient_ptr.size(); ++i) - if (transient_ptr[i]) { cactus_metal_free_shared(transient_ptr[i]); transient_ptr[i] = nullptr; } - }; auto metal_release = [&](size_t idx) { GraphNode& nd = *nodes_[idx]; if (aliases_input(nd)) nd.output_buffer.external_data = nullptr; @@ -1445,8 +1490,6 @@ void CactusGraph::execute(const std::string& profile_file) { since_flush = 0; since_recycle = 0; }; - struct CacheWordSnap { size_t idx; size_t word; uint64_t value; }; - std::vector kv_snapshot; for (size_t i = 0; i < n; ++i) { const GraphNode& nd = *nodes_[i]; bool kv = nd.op_type == OpType::KV_CACHE_APPEND; @@ -1459,12 +1502,6 @@ void CactusGraph::execute(const std::string& profile_file) { kv_snapshot.push_back({ci->second, 0, km[0]}); if (conv) kv_snapshot.push_back({ci->second, 1, km[1]}); } - auto kv_restore = [&]() { - for (const auto& s : kv_snapshot) { - uint64_t* km = reinterpret_cast(nodes_[s.idx]->output_buffer.get_data()); - if (km) km[s.word] = s.value; - } - }; auto input_metal_live = [&](const GraphNode& nd) -> bool { for (size_t id : nd.input_ids) { auto it = node_index_map_.find(id); @@ -1537,11 +1574,8 @@ void CactusGraph::execute(const std::string& profile_file) { continue; } cactus_metal_plan_disable(fplan); - cactus_metal_session_sync(); - kv_restore(); - release_transients(); - cactus_metal_set_active(false); - cactus_metal_session_end(); + metal_guard.armed = false; + metal_abort_cleanup(); execute(profile_file); return; } @@ -1563,11 +1597,8 @@ void CactusGraph::execute(const std::string& profile_file) { for (auto& fp : flipped) fp.first->precision = fp.second; if (!encoded && rplan && rplan[i] == 2) { metal_retype_disabled_ = true; - cactus_metal_session_sync(); - kv_restore(); - release_transients(); - cactus_metal_set_active(false); - cactus_metal_session_end(); + metal_guard.armed = false; + metal_abort_cleanup(); execute(profile_file); return; } @@ -1585,6 +1616,7 @@ void CactusGraph::execute(const std::string& profile_file) { if (encoded && ++since_flush >= 48) { cactus_metal_session_flush(); since_flush = 0; } if (encoded) maybe_recycle(); } + metal_guard.armed = false; release_transients(); cactus_metal_set_active(false); cactus_metal_session_end(); diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index f17ec19c7..07eeb8417 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -5,6 +5,8 @@ #include #include +bool cactus_kv_cache_grow(BufferDesc&, size_t, size_t); + namespace { bool is_alias_op(OpType op) { @@ -369,7 +371,9 @@ MetalFusePlan* cactus_metal_plan_build( gM *= nodes[(size_t)gate]->output_buffer.shape[d]; if (grb && gM == 1 && PrecisionTraits::is_cq(grb->precision) && grb->group_size == 128 && PrecisionTraits::cq_bits(grb->precision) == 4 - && !(grb->cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) { + && !(grb->cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) + && cactus_metal_transform_gemv_fits((uint32_t)( + grb->shape.size() >= 2 ? grb->shape[1] : grb->shape[0]))) { long gx = up_in((size_t)gate, 0); if (gx >= 0) { MetalCluster c7; @@ -1298,7 +1302,16 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, bool has_new = (c.u1 & 0x10000u) != 0; size_t ng = (hd + 31) / 32; size_t win = anchor.params.window_size; - bool sliding = win > 0; + bool sliding = win > 0 && win < nodes[c.b2]->params.max_cache_seq_len; + if (has_new && !sliding && clen >= mx) { + size_t ceiling = nodes[c.b2]->params.max_cache_seq_len; + if (!cactus_kv_cache_grow(kc, clen + 1, ceiling) + || !cactus_kv_cache_grow(vc, clen + 1, ceiling)) return false; + km = reinterpret_cast(kc.get_data()); + vm = reinterpret_cast(vc.get_data()); + if (!km || !vm) return false; + mx = km[1]; + } uint32_t Wn = sliding ? (uint32_t)(mx - sink - 1) : 0u; uint32_t Sn = sliding ? (uint32_t)sink : 0u; uint32_t Rn = (Wn > Sn) ? (Wn - Sn) : 1u; diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 0eb602c4f..29464ebfc 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -101,7 +101,6 @@ inline bool use_fp16_kv_cache() { } constexpr size_t kInitialCacheEntries = 256; -constexpr size_t kMetalInitialCacheEntries = 4000; inline bool kv_cache_resident() { return cactus_backend_metal(); @@ -120,11 +119,12 @@ inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { size_t total = fp16_cache ? fp16_cache_elements(new_max, kv_heads, hdim) : cache_buffer_size(new_max, kv_heads, hdim); BufferDesc resized({total}, fp16_cache ? Precision::FP16 : Precision::INT8); - bool shared = kv_cache_resident(); + const bool metal = kv_cache_resident(); void* old_data = buf.get_data(); - if (shared) { + if (metal) { + cactus_metal_session_sync(); void* p = cactus_metal_alloc_shared(resized.byte_size); - if (p) resized.set_external(p); else { resized.allocate(); shared = false; } + if (p) resized.set_external(p); else resized.allocate(); } else { resized.allocate(); } @@ -143,7 +143,7 @@ inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { } get_meta(resized)->max_seq_len = new_max; buf = std::move(resized); - if (shared) cactus_metal_free_shared(old_data); + cactus_metal_free_shared(old_data); return true; } @@ -177,7 +177,6 @@ void compute_kv_cache_state_node( size_t max_seq; if (sliding) max_seq = std::min(ceiling, window + node.params.cache_sink_size + 1); else if (num_slots > 1) max_seq = ceiling; - else if (kv_cache_resident()) max_seq = std::min(ceiling, kMetalInitialCacheEntries); else max_seq = std::min(ceiling, kInitialCacheEntries); size_t kv_heads = node.params.num_kv_heads; size_t hdim = node.params.head_dim; diff --git a/cactus-graph/tests/test_metal_parity.cpp b/cactus-graph/tests/test_metal_parity.cpp index 3181650b0..36a50a3eb 100644 --- a/cactus-graph/tests/test_metal_parity.cpp +++ b/cactus-graph/tests/test_metal_parity.cpp @@ -1,7 +1,9 @@ #include "test_utils.h" #include +#include #include #include +#include #include #include @@ -33,6 +35,13 @@ struct ParityCase { return input_shapes.size() - 1; } + size_t add_input_data(const std::vector& shape, std::vector<__fp16> data) { + input_shapes.push_back(shape); + input_data.push_back(std::move(data)); + input_data_f32.push_back({}); + return input_shapes.size() - 1; + } + size_t add_input_f32(const std::vector& shape, const std::vector& data) { input_shapes.push_back(shape); input_data.push_back({}); @@ -284,13 +293,209 @@ bool parity_cumsum() { return true; } +bool parity_softmax() { + ParityCase c; + c.add_input({5, 333}, -4.0f, 4.0f); + c.tolerance = 5e-3f; + c.build = [](CactusGraph& g, const std::vector& in) { return g.softmax(in[0], -1); }; + return c.check(); +} + +bool parity_rms_norm() { + for (size_t rows : {1u, 7u}) { + ParityCase c; + c.add_input({rows, 256}); + c.add_input({256}, 0.5f, 1.5f); + c.tolerance = 1e-2f; + c.build = [](CactusGraph& g, const std::vector& in) { + return g.rms_norm(in[0], in[1], 1e-5f); + }; + if (!c.check()) return false; + } + return true; +} + +bool parity_residual_rms_norm() { + for (size_t rows : {1u, 6u}) { + ParityCase c; + c.add_input({rows, 192}); + c.add_input({rows, 192}); + c.add_input({192}, 0.5f, 1.5f); + c.tolerance = 1e-2f; + c.build = [](CactusGraph& g, const std::vector& in) { + return g.rms_norm(g.add(in[0], in[1]), in[2], 1e-5f); + }; + if (!c.check()) return false; + } + return true; +} + +bool parity_elemwise_chain() { + ParityCase c; + c.add_input({4, 96}); + c.add_input({4, 96}); + c.add_input({4, 96}); + c.tolerance = 1e-2f; + c.build = [](CactusGraph& g, const std::vector& in) { + return g.add(g.multiply(g.silu(in[0]), in[1]), g.scalar_multiply(in[2], 0.5f)); + }; + return c.check(); +} + +bool parity_flash_attention() { + for (size_t T : {64u, 89u, 121u}) { + ParityCase c; + const size_t H = 2, D = 64, S = 729; + c.add_input({1, T, H, D}); + c.add_input({1, S, H, D}); + c.add_input({1, S, H, D}); + c.tolerance = 2e-2f; + c.build = [D](CactusGraph& g, const std::vector& in) { + return g.attention(in[0], in[1], in[2], 1.0f / std::sqrt((float)D), false); + }; + if (!c.check()) return false; + } + return true; +} + +bool parity_attention_causal() { + ParityCase c; + const size_t T = 37, H = 4, HKV = 2, D = 32; + c.add_input({1, T, H, D}); + c.add_input({1, T, HKV, D}); + c.add_input({1, T, HKV, D}); + c.tolerance = 2e-2f; + c.build = [D](CactusGraph& g, const std::vector& in) { + return g.attention(in[0], in[1], in[2], 1.0f / std::sqrt((float)D), true); + }; + return c.check(); +} + +bool parity_equality_exact() { + const size_t n = 2 * 31; + std::vector<__fp16> a = random_fp16(n, -1.0f, 1.0f, 7); + std::vector<__fp16> b = random_fp16(n, -1.0f, 1.0f, 8); + for (size_t i = 0; i < n; i += 2) b[i] = a[i]; + { + ParityCase c; + c.add_input_data({2, 31}, a); + c.add_input_data({2, 31}, b); + c.tolerance = 1e-3f; + c.build = [](CactusGraph& g, const std::vector& in) { return g.not_equal(in[0], in[1]); }; + if (!c.check()) return false; + } + { + std::vector<__fp16> z = a; + for (size_t i = 0; i < n; i += 2) z[i] = (__fp16)0.0f; + ParityCase c; + c.add_input_data({2, 31}, z); + c.tolerance = 1e-3f; + c.build = [](CactusGraph& g, const std::vector& in) { return g.scalar_not_equal(in[0], 0.0f); }; + if (!c.check()) return false; + } + return true; +} + +struct CacheStepData { + std::vector<__fp16> q, k, v; + size_t tokens; +}; + +std::vector run_cached_attention(const char* backend, size_t ceiling, size_t window, size_t sink, + size_t h, size_t kv, size_t d, + const std::vector& steps, size_t collect_from) { + if (cactus_backend_select(backend) != 0) return {}; + const size_t sentinel = std::numeric_limits::max(); + const float scale = 1.0f / std::sqrt((float)d); + std::vector collected; + CactusGraph g; + size_t kc = g.kv_cache_state(ceiling, kv, d, window, sink); + size_t vc = g.kv_cache_state(ceiling, kv, d, window, sink); + for (size_t si = 0; si < steps.size(); ++si) { + const auto& data = steps[si]; + size_t s = data.tokens; + size_t iq = g.input({1, s, h, d}, Precision::FP16); + size_t ik = g.input({1, s, kv, d}, Precision::FP16); + size_t iv = g.input({1, s, kv, d}, Precision::FP16); + g.set_input(iq, const_cast<__fp16*>(data.q.data()), Precision::FP16); + g.set_input(ik, const_cast<__fp16*>(data.k.data()), Precision::FP16); + g.set_input(iv, const_cast<__fp16*>(data.v.data()), Precision::FP16); + g.kv_cache_append(ik, kc, window, sink); + g.kv_cache_append(iv, vc, window, sink); + size_t attn = g.attention_cached(iq, ik, iv, kc, vc, scale, sentinel, window); + g.execute(); + if (si >= collect_from) { + const __fp16* p = static_cast(g.get_output(attn)); + for (size_t i = 0; i < s * h * d; ++i) collected.push_back((float)p[i]); + } + g.soft_reset(); + } + g.hard_reset(); + cactus_backend_select("auto"); + return collected; +} + +bool check_cached_attention(size_t ceiling, size_t window, size_t sink, + const std::vector& chunk_sizes, size_t decode_steps, + float tolerance, size_t collect_from = 0) { + const size_t h = 4, kv = 2, d = 32; + std::vector steps; + uint32_t seed = 100; + for (size_t s : chunk_sizes) { + steps.push_back({random_fp16(s * h * d, -1.0f, 1.0f, seed), + random_fp16(s * kv * d, -1.0f, 1.0f, seed + 1), + random_fp16(s * kv * d, -1.0f, 1.0f, seed + 2), s}); + seed += 3; + } + for (size_t i = 0; i < decode_steps; ++i) { + steps.push_back({random_fp16(h * d, -1.0f, 1.0f, seed), + random_fp16(kv * d, -1.0f, 1.0f, seed + 1), + random_fp16(kv * d, -1.0f, 1.0f, seed + 2), 1}); + seed += 3; + } + std::vector cpu = run_cached_attention("cpu", ceiling, window, sink, h, kv, d, steps, collect_from); + std::vector metal = run_cached_attention("metal", ceiling, window, sink, h, kv, d, steps, collect_from); + if (cpu.size() != metal.size() || cpu.empty()) return false; + for (size_t i = 0; i < cpu.size(); ++i) { + float scale = std::max(1.0f, std::fabs(cpu[i])); + if (std::fabs(cpu[i] - metal[i]) > tolerance * scale) { + std::cout << " mismatch at " << i << " (token " << i / (4 * 32) << "): cpu=" << cpu[i] + << " metal=" << metal[i] << "\n"; + return false; + } + } + return true; +} + +bool parity_sliding_window_cache() { + if (!check_cached_attention(/*ceiling=*/64, /*window=*/8, /*sink=*/2, + /*chunks=*/{5}, /*decode_steps=*/12, 5e-2f)) return false; + return check_cached_attention(/*ceiling=*/64, /*window=*/8, /*sink=*/2, + /*chunks=*/{4, 8}, /*decode_steps=*/6, 5e-2f, /*collect_from=*/2); +} + +bool parity_cache_growth() { + return check_cached_attention(/*ceiling=*/2048, /*window=*/0, /*sink=*/2, + /*chunks=*/{600}, /*decode_steps=*/3, 5e-2f); +} + } int main() { TestRunner runner("Metal Parity (CPU vs Metal per-op)"); if (!metal_present()) { +#if defined(__APPLE__) + if (std::getenv("CACTUS_PARITY_ALLOW_SKIP")) { + std::cout << "Metal backend unavailable; skipping parity suite (CACTUS_PARITY_ALLOW_SKIP).\n"; + return 0; + } + std::cout << "FAIL: Metal backend unavailable on Apple hardware " + "(set CACTUS_PARITY_ALLOW_SKIP=1 to skip).\n"; + return 1; +#else std::cout << "Metal backend unavailable; skipping parity suite.\n"; return 0; +#endif } runner.run_test("abs", parity_unary(&CactusGraph::abs)); @@ -329,6 +534,15 @@ int main() { runner.run_test("conv2d_pointwise_1x1", parity_conv2d(&CactusGraph::conv2d_pointwise_1x1, false, true)); runner.run_test("batchnorm", parity_batchnorm()); runner.run_test("groupnorm", parity_groupnorm()); + runner.run_test("softmax", parity_softmax()); + runner.run_test("rms_norm", parity_rms_norm()); + runner.run_test("residual_rms_norm (fused)", parity_residual_rms_norm()); + runner.run_test("elemwise_chain (fused)", parity_elemwise_chain()); + runner.run_test("flash_attention (T%64!=0)", parity_flash_attention()); + runner.run_test("attention_causal", parity_attention_causal()); + runner.run_test("equality_exact", parity_equality_exact()); + runner.run_test("sliding_window_cache (ring wrap)", parity_sliding_window_cache()); + runner.run_test("cache_growth", parity_cache_growth()); runner.print_summary(); return runner.all_passed() ? 0 : 1; diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 858fb381d..445cdeb1a 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -60,6 +60,7 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* const* Ws, int B, void* const* codes); bool cactus_metal_encode_gemv_precoded(void* out, const void* code, const CactusQuantMatrix* W); bool cactus_metal_encode_transform_gemv(void* out, const void* x, const CactusQuantMatrix* W, const void* osw); +bool cactus_metal_transform_gemv_fits(uint32_t K); bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, const CactusQuantMatrix* const* Ws, int B); bool cactus_metal_encode_swiglu_transform(void* code, const void* gate, const void* up, diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 6f4395f18..03488338b 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -849,13 +849,13 @@ kernel void emb_ortho_m(device const uchar* packed [[buffer(0)]], device const h float acc = 0; for (uint k0=0; k0(0.0h); + simdgroup_float8x8 of[8]; + for (uint d = 0; d < 8; ++d) of[d] = make_filled_simdgroup_matrix(0.0f); for (uint kb = 0; kb < S; kb += BK) { for (uint x = tid; x < BK * D / 4; x += NT) { @@ -1714,7 +1714,7 @@ kernel void attn_flash_f16(device const half* q [[buffer(0)]], device const half Ms[sgid][r] = mnew; Ls[sgid][r] = Ls[sgid][r] * alpha + sloc; if (mnew != mold && !first) Rf[sgid] = 1; - Dg[sgid][r * 8 + r] = (half)alpha; + Dg[sgid][r * 8 + r] = alpha; } simdgroup_barrier(mem_flags::mem_threadgroup); @@ -1723,10 +1723,10 @@ kernel void attn_flash_f16(device const half* q [[buffer(0)]], device const half for (uint j = 0; j < 4; ++j) simdgroup_load(pf[j], Ps[sgid] + j * 8, BK); if (rescale) { - simdgroup_half8x8 dgf; + simdgroup_float8x8 dgf; simdgroup_load(dgf, Dg[sgid], 8); for (uint d = 0; d < 8; ++d) { - simdgroup_half8x8 acc; + simdgroup_float8x8 acc; simdgroup_multiply(acc, dgf, of[d]); for (uint j = 0; j < 4; ++j) { simdgroup_half8x8 vf; @@ -1751,21 +1751,33 @@ kernel void attn_flash_f16(device const half* q [[buffer(0)]], device const half uint r = lane / 4; if ((lane % 4) == 0) { float l = Ls[sgid][r]; - Dg[sgid][r * 8 + r] = (half)(l > 0.0f ? 1.0f / l : 0.0f); + Dg[sgid][r * 8 + r] = l > 0.0f ? 1.0f / l : 0.0f; } simdgroup_barrier(mem_flags::mem_threadgroup); - simdgroup_half8x8 dgf; + simdgroup_float8x8 dgf; simdgroup_load(dgf, Dg[sgid], 8); if (interior) { for (uint d = 0; d < 8; ++d) { simdgroup_multiply(of[d], dgf, of[d]); - simdgroup_store(of[d], y + ((size_t)row0 * H + h) * D + d * 8, (size_t)H * D); + simdgroup_store(of[d], Ss[sgid], 8); + simdgroup_barrier(mem_flags::mem_threadgroup); + for (uint e = lane; e < 64; e += 32) { + uint r2 = e / 8, cc = e % 8; + y[((size_t)(row0 + r2) * H + h) * D + d * 8 + cc] = (half)Ss[sgid][e]; + } + simdgroup_barrier(mem_flags::mem_threadgroup); } } else { threadgroup half* Oe = KVs; for (uint d = 0; d < 8; ++d) { simdgroup_multiply(of[d], dgf, of[d]); - simdgroup_store(of[d], Oe + (size_t)sgid * 8 * D + d * 8, D); + simdgroup_store(of[d], Ss[sgid], 8); + simdgroup_barrier(mem_flags::mem_threadgroup); + for (uint e = lane; e < 64; e += 32) { + uint r2 = e / 8, cc = e % 8; + Oe[(size_t)sgid * 8 * D + r2 * D + d * 8 + cc] = (half)Ss[sgid][e]; + } + simdgroup_barrier(mem_flags::mem_threadgroup); } threadgroup_barrier(mem_flags::mem_threadgroup); for (uint x = tid; x < BQ * D; x += NT) { @@ -1796,7 +1808,7 @@ kernel void gemm_batch_f32a(device const float* a [[buffer(0)]], device const ha device const float* ar = a + (size_t)bidx*M*K + (size_t)m*K; device const half* br = b + (size_t)bidx*K*N + nn; float acc = 0; - for (uint k = 0; k < K; ++k) acc = fma((float)((half)ar[k]), (float)br[(size_t)k*N], acc); + for (uint k = 0; k < K; ++k) acc = fma(ar[k], (float)br[(size_t)k*N], acc); size_t oi = (size_t)bidx*M*N + mn; if (f32out != 0u) ((device float*)y)[oi] = acc; else ((device half*)y)[oi] = (half)acc; @@ -1943,23 +1955,26 @@ kernel void rope_full_f16(device const half* x [[buffer(0)]], device half* y [[b constant uint& pos0 [[buffer(6)]], constant float& theta [[buffer(7)]], constant uint& gptj [[buffer(8)]], uint2 gid [[thread_position_in_grid]]) { - uint d = gid.x, tok = gid.y; - if (d >= D) return; - size_t off = (size_t)tok * D + d; - uint seq = (tok / H) % S; - if (d >= rot) { y[off] = x[off]; return; } + uint idx = gid.x, tok = gid.y; uint hr = rot / 2; - uint pair_i = gptj ? d / 2 : (d % hr); - float freq = 1.0f / precise::pow(theta, (2.0f * pair_i) / (float)rot); + if (idx >= hr) { + uint d = rot + (idx - hr); + if (d >= D) return; + size_t off = (size_t)tok * D + d; + y[off] = x[off]; + return; + } + uint seq = (tok / H) % S; + float freq = 1.0f / precise::pow(theta, (2.0f * idx) / (float)rot); float angle = (float)(pos0 + seq) * freq; float c = (float)(half)precise::cos(angle); float sn = (float)(half)precise::sin(angle); - float v = (float)x[off]; - float other; - bool first; - if (gptj) { first = (d % 2) == 0; other = (float)x[off + (first ? 1 : -1)]; } - else { first = d < hr; other = (float)x[off + (first ? (long)hr : -(long)hr)]; } - y[off] = (half)(first ? v*c - other*sn : v*c + other*sn); + uint d0 = gptj ? 2u * idx : idx; + uint d1 = gptj ? 2u * idx + 1u : idx + hr; + size_t off0 = (size_t)tok * D + d0, off1 = (size_t)tok * D + d1; + float v0 = (float)x[off0], v1 = (float)x[off1]; + y[off0] = (half)(v0*c - v1*sn); + y[off1] = (half)(v1*c + v0*sn); } kernel void maxpool1d_f16(device const half* x [[buffer(0)]], device half* y [[buffer(1)]], @@ -3124,6 +3139,7 @@ kernel void kv_append_ring_i8_m(device const half* src [[buffer(0)]], device cha uint pos = current_len + i; uint R = (W > sink) ? (W - sink) : 1u; uint slot = (pos < W) ? pos : sink + ((pos - sink) % R); + if (pos >= sink && pos + R < current_len + M) return; device char* dst = int8base + (size_t)slot*i8s + (size_t)h*hdim + gstart; for (uint k=0;k #include #include +#include #include "cactus_kernels_msl.h" @@ -137,6 +138,7 @@ float cb_scale=1.f; id wf16=nil; bool wf16_volatile=false; + uint64_t wf16_tick=0; }; std::unordered_map g_resident; @@ -153,6 +155,8 @@ static uint64_t resident_key(const CactusQuantMatrix* W) { return h; } bool g_wf16_live = false; +size_t g_wf16_bytes = 0; +uint64_t g_wf16_clock = 0; id g_attn_scores = nil; std::unordered_map g_resident_emb; std::mutex g_resident_mu; @@ -268,6 +272,35 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, static const uint32_t g_mr_rows = 8u * 2u; static const bool g_predequant=(std::getenv("CACTUS_NO_PREDEQUANT")==nullptr); +inline bool tg_mem_ok(size_t bytes) { + return bytes <= (size_t)[ctx().dev maxThreadgroupMemoryLength]; +} + +size_t wf16_budget() { + static const size_t cap = [] { + if (const char* e = std::getenv("CACTUS_WF16_CAP_MB")) return (size_t)atoll(e) << 20; + return (size_t)([ctx().dev recommendedMaxWorkingSetSize] / 4); + }(); + return cap; +} + +void wf16_evict_for(size_t need, const ResW* keep) { + std::lock_guard lk(g_resident_mu); + const size_t cap = wf16_budget(); + while (g_wf16_bytes + need > cap) { + ResW* victim = nullptr; + for (auto& kv : g_resident) { + ResW& r = kv.second; + if (!r.wf16 || &r == keep) continue; + if (!victim || r.wf16_tick < victim->wf16_tick) victim = &r; + } + if (!victim) break; + g_wf16_bytes -= (size_t)victim->wf16.length; + victim->wf16 = nil; + victim->wf16_volatile = false; + } +} + id ensureEncoder() { if (!g_enc) { if (!g_cmd) g_cmd = [ctx().queue commandBuffer]; @@ -375,22 +408,32 @@ void cactus_metal_session_begin() { if (!g_arpool) g_arpool = objc_autoreleasePoolPush(); } static id g_last_cmd = nil; +static std::atomic g_cmd_failed{false}; +static void watch_cmd_errors(id cmd) { + [cmd addCompletedHandler:^(id cb) { + if (cb.status == MTLCommandBufferStatusError) { + g_cmd_failed.store(true, std::memory_order_relaxed); + fprintf(stderr, "[cactus-metal] command buffer failed: %s\n", + cb.error ? [[cb.error localizedDescription] UTF8String] : "unknown"); + } + }]; +} +static void reclaim_dead_slabs(); void cactus_metal_session_sync() { if (g_enc) { [g_enc endEncoding]; g_enc = nil; } if (g_cmd) { + watch_cmd_errors(g_cmd); [g_cmd commit]; g_last_cmd = g_cmd; g_cmd = nil; } if (g_last_cmd) { [g_last_cmd waitUntilCompleted]; - if (g_last_cmd.status == MTLCommandBufferStatusError) - fprintf(stderr, "[cactus-metal] command buffer failed: %s\n", - g_last_cmd.error ? [[g_last_cmd.error localizedDescription] UTF8String] : "unknown"); g_last_cmd = nil; } for (id b : g_pending) g_free[(size_t)b.length].push_back(b); g_pending.clear(); + reclaim_dead_slabs(); } void cactus_metal_session_end() { cactus_metal_session_sync(); @@ -405,11 +448,12 @@ void cactus_metal_invalidate_host_wraps() { g_resident_emb.clear(); g_dequant_cache.clear(); g_mps_mats.clear(); + g_wf16_bytes = 0; } void cactus_metal_session_flush() { if (g_enc) { [g_enc endEncoding]; g_enc = nil; } - if (g_cmd) { [g_cmd commit]; g_last_cmd = g_cmd; g_cmd = nil; } + if (g_cmd) { watch_cmd_errors(g_cmd); [g_cmd commit]; g_last_cmd = g_cmd; g_cmd = nil; } } @@ -425,30 +469,54 @@ void cactus_metal_session_flush() { return c; } +struct SlabInfo { size_t cap = 0, used = 0, live = 0; }; static char* g_slab = nullptr; -static size_t g_slab_used = 0, g_slab_cap = 0; -static std::unordered_set g_slab_bases; +static std::map g_slabs; + +static void reclaim_dead_slabs() { + for (auto it = g_slabs.begin(); it != g_slabs.end();) { + SlabInfo& s = it->second; + if (s.live != 0 || s.used == 0) { ++it; continue; } + if ((char*)it->first == g_slab) { s.used = 0; ++it; continue; } + g_shared.erase(it->first); + it = g_slabs.erase(it); + } +} + void* cactus_metal_alloc_pooled(size_t bytes) { if (!ctx().ok) return nullptr; if (bytes > (4u << 20)) return cactus_metal_alloc_shared(bytes); size_t need = (bytes + 255) & ~size_t(255); - if (!g_slab || g_slab_used + need > g_slab_cap) { + SlabInfo* cur = nullptr; + if (g_slab) { + auto it = g_slabs.find(reinterpret_cast(g_slab)); + if (it != g_slabs.end()) cur = &it->second; + } + if (!cur || cur->used + need > cur->cap) { size_t cap = 32u << 20; id b = [ctx().dev newBufferWithLength:cap options:MTLResourceStorageModeShared]; if (!b) return cactus_metal_alloc_shared(bytes); g_slab = (char*)[b contents]; - g_slab_used = 0; - g_slab_cap = cap; g_shared[reinterpret_cast(g_slab)] = b; - g_slab_bases.insert(reinterpret_cast(g_slab)); + cur = &g_slabs[reinterpret_cast(g_slab)]; + *cur = SlabInfo{cap, 0, 0}; } - void* p = g_slab + g_slab_used; - g_slab_used += need; + void* p = g_slab + cur->used; + cur->used += need; + cur->live += 1; return p; } void cactus_metal_free_shared(void* contents) { - if (g_slab_bases.count(reinterpret_cast(contents))) return; - auto it = g_shared.find(reinterpret_cast(contents)); + uintptr_t a = reinterpret_cast(contents); + auto sit = g_slabs.upper_bound(a); + if (sit != g_slabs.begin()) { + --sit; + if (a < sit->first + sit->second.cap) { + if (sit->second.live > 0) --sit->second.live; + return; + } + } + auto it = g_shared.find(a); if (it != g_shared.end()) { g_pending.push_back(it->second); g_shared.erase(it); } } @@ -701,6 +769,7 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu size_t code_bytes = (size_t)ng*gs*sizeof(__fp16); if (!g_code_buf || (size_t)g_code_buf.length < code_bytes) g_code_buf = [ctx().dev newBufferWithLength:code_bytes options:MTLResourceStorageModeShared]; + if (!g_code_buf) return false; ensureEncoder(); bool simdT = (gs==128u && ctx().psoTsimd); [g_enc setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; @@ -784,10 +853,15 @@ bool cactus_metal_encode_gemv_precoded(void* out, const void* code, const Cactus return true; } +bool cactus_metal_transform_gemv_fits(uint32_t K) { + return ctx().ok && tg_mem_ok((size_t)K*2 + 8*128*sizeof(float) + 64); +} + bool cactus_metal_encode_transform_gemv(void* out, const void* x, const CactusQuantMatrix* W, const void* osw) { if (!ctx().ok || !ctx().psoMega) return false; const uint32_t gs=W->group_size, K=W->K, ng=W->num_groups, N=W->N; if (gs != 128u || !quant_fast_eligible(W) || (N % 16u) != 0u) return false; + if (!cactus_metal_transform_gemv_fits(K)) return false; ResW& rw = resident(W); ensureEncoder(); struct { uint32_t K, ng, oswi; } U = { K, ng, (uint32_t)(osw ? 1 : 0) }; @@ -876,6 +950,7 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus size_t code_bytes = (size_t)M*ng*gs*sizeof(__fp16); if (!g_code_buf_m || (size_t)g_code_buf_m.length < code_bytes) g_code_buf_m = [ctx().dev newBufferWithLength:code_bytes options:MTLResourceStorageModeShared]; + if (!g_code_buf_m) return false; ensureEncoder(); [g_enc setComputePipelineState:ctx().psoTm]; @@ -886,15 +961,26 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; - if ((g_predequant && ctx().psoGdense && ctx().psoDeq) || lowbit) { - bool need_dequant = false; + bool dense = (g_predequant && ctx().psoGdense && ctx().psoDeq) || lowbit; + bool need_dequant = false; + if (dense) { if (!rw.wf16) { - rw.wf16 = [ctx().dev newBufferWithLength:(size_t)N*K*2 options:MTLResourceStorageModeShared]; + size_t wbytes = (size_t)N*K*2; + wf16_evict_for(wbytes, &rw); + rw.wf16 = [ctx().dev newBufferWithLength:wbytes options:MTLResourceStorageModeShared]; + if (rw.wf16) g_wf16_bytes += (size_t)rw.wf16.length; need_dequant = true; } else if (rw.wf16_volatile) { need_dequant = [rw.wf16 setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty; } + if (!rw.wf16) { + if (lowbit) return false; + dense = false; + } + } + if (dense) { rw.wf16_volatile = false; + rw.wf16_tick = ++g_wf16_clock; g_wf16_live = true; if (need_dequant) { [g_enc setComputePipelineState:(lowbit?ctx().psoDeqLow:ctx().psoDeq)]; @@ -967,6 +1053,7 @@ bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQu if (!ctx().ok || !W->rotation || (W->bits != 4 && W->bits != 2 && W->bits != 3)) return false; const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size, bits=W->bits; if (ng != 1 || gs != K) return false; + if (!tg_mem_ok((size_t)K*sizeof(float))) return false; if (bits != 4 && !(ctx().psoEmbOW && (K % 8u) == 0u)) return false; ResW& rw = resident(W); if (!rw.packed || !rw.rotation || !rw.codebook || !rw.norms || !rw.recip) return false; @@ -1635,7 +1722,9 @@ bool cactus_metal_encode_rope_full(void* out, const void* in, uint32_t tokens, u [g_enc setBytes:&D length:4 atIndex:4]; [g_enc setBytes:&rot length:4 atIndex:5]; [g_enc setBytes:&pos0 length:4 atIndex:6]; [g_enc setBytes:&theta length:4 atIndex:7]; [g_enc setBytes:&gj length:4 atIndex:8]; - [g_enc dispatchThreads:MTLSizeMake(D, tokens, 1) threadsPerThreadgroup:MTLSizeMake(std::min(D, 64u), 4, 1)]; + uint32_t gx = rot/2 + (D > rot ? D - rot : 0); + if (gx == 0) gx = 1; + [g_enc dispatchThreads:MTLSizeMake(gx, tokens, 1) threadsPerThreadgroup:MTLSizeMake(std::min(gx, 64u), 4, 1)]; return true; } bool cactus_metal_encode_maxpool1d(void* out, const void* in, uint32_t NC, uint32_t L, diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index fb5efdeb3..0f7daa4f9 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -9,6 +9,7 @@ bool cactus_metal_encode_softcap(void*, const void*, size_t, float) { return fal bool cactus_metal_encode_adjust_logits(void*, size_t, const uint32_t*, uint32_t, int64_t, float) { return false; } bool cactus_metal_encode_transform_gemv(void*, const void*, const CactusQuantMatrix*, const void*) { return false; } +bool cactus_metal_transform_gemv_fits(uint32_t) { return false; } bool cactus_metal_encode_gemv_cat(void* const*, const void* const*, const CactusQuantMatrix* const*, int) { return false; } bool cactus_metal_encode_swiglu_transform(void*, const void*, const void*, From 240304f99f0ddb30035b487d46ce9b842709699b Mon Sep 17 00:00:00 2001 From: jakmro Date: Sat, 4 Jul 2026 16:02:44 +0200 Subject: [PATCH 27/36] fixes --- README.md | 14 ++++++------ cactus-engine/src/engine.h | 6 ++--- cactus-engine/src/utils.h | 2 +- cactus-kernels/src/metal_backend.mm | 12 +++++++++- docs/cactus_engine.md | 7 +++--- docs/cactus_transpiler.md | 2 +- python/README.md | 4 ++-- python/cactus/cli/__init__.py | 35 ++++++++++++++--------------- python/cactus/cli/code.py | 4 ++-- python/cactus/cli/common.py | 5 +++-- python/cactus/cli/convert.py | 4 ++-- python/cactus/cli/download.py | 24 +++++++------------- python/cactus/cli/model.py | 6 ++--- python/cactus/cli/upload.py | 4 ++-- python/tests/test_bindings.py | 12 +++++----- python/tests/test_cli_run.py | 3 ++- python/tests/test_model.py | 4 ++-- 17 files changed, 76 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index b44e39712..1972ee094 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ graph.hard_reset(); │ │ │ cactus run [model|path] run a model (downloads if needed) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --backend auto|cpu|metal inference backend (default: auto) │ │ --image image file for VLM inference │ │ --audio audio file for audio chat │ @@ -210,20 +210,20 @@ graph.hard_reset(); │ --file audio file to transcribe (WAV) │ │ --language language code (default: en) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ │ │ cactus download [model] get a bundle (prebuilt, else build) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ │ │ cactus convert [dir] HuggingFace -> runnable cactus bundle │ │ (CQ weights + runtime graph) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ --lora merge a LoRA adapter before converting │ @@ -234,7 +234,7 @@ graph.hard_reset(); │ --host bind address (default: 127.0.0.1) │ │ --port port (default: 8080) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --backend auto|cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ @@ -245,7 +245,7 @@ graph.hard_reset(); │ cactus code run the AI coding agent (TUI / print) │ │ --serve-model auto-start a server with this model │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --backend auto|cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ @@ -270,7 +270,7 @@ graph.hard_reset(); │ --model default: LiquidAI/LFM2-VL-450M │ │ --transcription-model default: openai/whisper-base │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild of test models │ │ --suite run a single test suite by name │ diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 220f3595a..d0cffb294 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -637,7 +637,7 @@ class Model { uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.0f); + float min_p = 0.15f, float repetition_penalty = 1.1f); void set_sample_seed(uint64_t seed) { sample_rng_.seed(static_cast(seed)); } bool prefill_and_sample_first_token(const std::vector& tokens, uint32_t& out_token, float* out_uncertainty = nullptr); @@ -669,13 +669,13 @@ class Model { uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.0f); + float min_p = 0.15f, float repetition_penalty = 1.1f); uint32_t decode_with_audio(const std::vector& tokens, const std::vector>& audio_features_per_message, float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.0f, + float min_p = 0.15f, float repetition_penalty = 1.1f, float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); struct ParakeetTdtStreamState { diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index b0aaed8ea..afc720a1b 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -507,7 +507,7 @@ struct InferenceOptions { float temperature = 0.0f; float top_p = 0.0f; float min_p = 0.15f; - float repetition_penalty = 1.0f; + float repetition_penalty = 1.1f; uint64_t sample_seed = 0; float confidence_threshold = -1.0f; size_t top_k = 0; diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 565311af4..d18f31712 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -6,7 +6,10 @@ #import #import +#include +#if TARGET_OS_OSX #include +#endif #import #include #include @@ -279,7 +282,9 @@ inline bool tg_mem_ok(size_t bytes) { size_t wf16_budget() { static const size_t cap = [] { if (const char* e = std::getenv("CACTUS_WF16_CAP_MB")) return (size_t)atoll(e) << 20; - return (size_t)([ctx().dev recommendedMaxWorkingSetSize] / 4); + if (@available(macOS 10.12, iOS 16.0, *)) + return (size_t)([ctx().dev recommendedMaxWorkingSetSize] / 4); + return (size_t)([NSProcessInfo processInfo].physicalMemory / 4); }(); return cap; } @@ -334,6 +339,10 @@ void wf16_evict_for(size_t need, const ResW* keep) { } size_t wraplen = (need + 16383u) & ~(size_t)16383u; uintptr_t lo = base, hi = base + wraplen; +#if TARGET_OS_OSX + // Expand the wrap to the whole VM region so neighboring host pointers hit + // the same MTLBuffer. mach_vm.h is macOS-only; elsewhere the page-aligned + // span above is used as-is (correct, just more wrap entries). { mach_vm_address_t raddr = a; mach_vm_size_t rsize = 0; @@ -350,6 +359,7 @@ void wf16_evict_for(size_t need, const ResW* keep) { } if (obj != MACH_PORT_NULL) mach_port_deallocate(mach_task_self(), obj); } +#endif auto ov = g_wrapped.lower_bound(lo); if (ov != g_wrapped.begin()) { auto prev = std::prev(ov); diff --git a/docs/cactus_engine.md b/docs/cactus_engine.md index 6d2f20d7b..00d8dc6e4 100644 --- a/docs/cactus_engine.md +++ b/docs/cactus_engine.md @@ -25,9 +25,10 @@ cactus auth `cactus download` fetches a **pre-built runtime bundle** (CQ weights + serialized graph + manifest) from [huggingface.co/Cactus-Compute](https://huggingface.co/Cactus-Compute) into -`weights/-cq[-]/`. Defaults to `--platform auto` (the -best build for your host, e.g. Apple Silicon on macOS); pass `--platform cpu` -for the generic build. The result can be loaded directly via `cactus_init()`. +`weights/-cq[-]/`. Defaults to `--weights general` (the +portable build, used on every platform); pass `--weights apple` for the +Apple-specific (CoreML/NPU) variant. The result can be loaded directly via +`cactus_init()`. For models not on Cactus-Compute, build a bundle from source with `cactus convert ` (quantizes the weights and builds the runtime graph). diff --git a/docs/cactus_transpiler.md b/docs/cactus_transpiler.md index 02c128a72..3be3c77ef 100644 --- a/docs/cactus_transpiler.md +++ b/docs/cactus_transpiler.md @@ -454,7 +454,7 @@ huggingface.co/Cactus-Compute) or a local path to a bundle directory. | Option | Description | |--------|-------------| | `--bits 1\|2\|3\|4` | CQ quantization bits when downloading (default: 4) | -| `--platform auto\|cpu\|apple` | Target platform (default: auto — best for the host) | +| `--weights general\|apple` | Weights bundle variant (default: general — portable on every platform) | | `--token ` | HuggingFace token (gated models) | | `--prompt ` | Input prompt | | `--input-ids ` | Comma-separated token IDs | diff --git a/python/README.md b/python/README.md index a259aa8ce..6bf7e0dff 100644 --- a/python/README.md +++ b/python/README.md @@ -20,9 +20,9 @@ cactus build --python ```bash -# Download pre-built bundles (defaults to --platform auto; use --platform cpu for the generic build) +# Download pre-built bundles (defaults to --weights general, the portable build) cactus download LiquidAI/LFM2-VL-450M -cactus download openai/whisper-small --platform apple # CoreML/NPU variant +cactus download openai/whisper-small --weights apple # CoreML/NPU variant # Optional: set your Cactus Cloud API key for automatic cloud fallback cactus auth diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index a7a394139..708477609 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -7,17 +7,16 @@ DEFAULT_TRANSCRIPTION_MODEL_ID, DEFAULT_TEST_MODEL_ID, DEFAULT_TEST_TRANSCRIPTION_MODEL_ID, - SUPPORTED_PLATFORMS, + SUPPORTED_WEIGHTS_VARIANTS, ) from .download import cmd_download -_PLATFORM_CHOICES = ("auto", "cpu", *SUPPORTED_PLATFORMS) -_PLATFORM_HELP = ( - f"target platform: auto = best for this host, e.g. apple on macOS (default); " - f"cpu = generic ARM; " - f"or one of: {', '.join(SUPPORTED_PLATFORMS) if SUPPORTED_PLATFORMS else '(none yet)'}" +_WEIGHTS_CHOICES = ("general", *SUPPORTED_WEIGHTS_VARIANTS) +_WEIGHTS_HELP = ( + f"weights bundle variant: general = portable bundle for every platform (default); " + f"or a vendor variant: {', '.join(SUPPORTED_WEIGHTS_VARIANTS) if SUPPORTED_WEIGHTS_VARIANTS else '(none yet)'}" ) -_PLATFORM_PIPE = "|".join(_PLATFORM_CHOICES) +_WEIGHTS_PIPE = "|".join(_WEIGHTS_CHOICES) from .compile import cmd_build from .serve import cmd_serve from .transcribe import cmd_transcribe @@ -45,8 +44,8 @@ def _build_parent(): p = argparse.ArgumentParser(add_help=False) p.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, help="CQ quantization (default: 4)") - p.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help=_PLATFORM_HELP) + p.add_argument("--weights", choices=_WEIGHTS_CHOICES, default="general", + help=_WEIGHTS_HELP) p.add_argument("--token", help="HuggingFace token") p.add_argument("--reconvert", action="store_true", help="Force local rebuild from source") @@ -111,7 +110,7 @@ def create_parser(): cactus run [model|path] run a model (default: {DEFAULT_MODEL_ID}) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --image image file for VLM inference --audio audio file for audio chat --system system prompt @@ -124,19 +123,19 @@ def create_parser(): --file audio file to transcribe (WAV) --language language code (default: en) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (gated models) --reconvert force local rebuild from source cactus download [model] fetch a prebuilt bundle, else build locally (default: {DEFAULT_MODEL_ID}) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (gated models) --reconvert force local rebuild from source cactus convert [dir] build a runnable bundle locally (skips prebuilt fetch) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (gated models) --reconvert force local rebuild from source --lora merge a LoRA adapter before converting @@ -145,7 +144,7 @@ def create_parser(): --host bind address (default: 127.0.0.1) --port port (default: 8080) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (gated models) --reconvert force local rebuild from source --no-cloud-handoff disable automatic cloud handoff @@ -165,7 +164,7 @@ def create_parser(): --model default: {DEFAULT_TEST_MODEL_ID} --transcription-model default: {DEFAULT_TEST_TRANSCRIPTION_MODEL_ID} --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (gated models) --reconvert force local rebuild of test models --suite run a single test suite from any @@ -189,7 +188,7 @@ def create_parser(): (quantizes weights to CQ, then builds the runtime graph) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (defaults to $HF_TOKEN) --reconvert force weight conversion from source --lora merge a LoRA adapter before converting @@ -379,8 +378,8 @@ def create_parser(): help="Output directory (default: weights/)") convert_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, help="CQ quantization bits (default: 4)") - convert_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help=_PLATFORM_HELP) + convert_parser.add_argument("--weights", choices=_WEIGHTS_CHOICES, default="general", + help=_WEIGHTS_HELP) convert_parser.add_argument("--token", help="HuggingFace token") convert_parser.add_argument("--lora", help="Path or HF id of a LoRA adapter to merge before converting (requires `peft`)") diff --git a/python/cactus/cli/code.py b/python/cactus/cli/code.py index 96adfe2c7..b8362ebf2 100644 --- a/python/cactus/cli/code.py +++ b/python/cactus/cli/code.py @@ -130,7 +130,7 @@ def cmd_code(args) -> int: model_flags_given = ( getattr(args, "reconvert", False) or getattr(args, "bits", 4) != 4 - or getattr(args, "platform", "auto") != "auto" + or getattr(args, "weights", "general") != "general" or bool(getattr(args, "token", None)) ) if args.serve_model: @@ -146,7 +146,7 @@ def cmd_code(args) -> int: sys.executable, "-m", "cactus", "serve", serve_model, "--host", args.host, "--port", str(args.port), "--no-access-log", "--bits", str(getattr(args, "bits", 4)), - "--platform", getattr(args, "platform", "auto"), + "--weights", getattr(args, "weights", "general"), "--backend", getattr(args, "backend", "auto"), ] if getattr(args, "token", None): diff --git a/python/cactus/cli/common.py b/python/cactus/cli/common.py index 71601e4cb..569994792 100644 --- a/python/cactus/cli/common.py +++ b/python/cactus/cli/common.py @@ -39,8 +39,9 @@ def transpiled_root() -> Path: DEFAULT_TEST_TRANSCRIPTION_MODEL_ID = "openai/whisper-base" -# Add a new vendor platform by appending its name here. -SUPPORTED_PLATFORMS: tuple[str, ...] = ("apple",) +# Vendor-specific weight-bundle variants; add a new vendor by appending its +# name here. "general" (the portable bundle) is always available and default. +SUPPORTED_WEIGHTS_VARIANTS: tuple[str, ...] = ("apple",) RED = '\033[0;31m' diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index 55cf3333e..6b9191d26 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -3,7 +3,7 @@ from pathlib import Path from .common import GREEN, RED, YELLOW, print_color -from .download import get_bundle_dir, get_weights_dir, resolve_platform +from .download import get_bundle_dir, get_weights_dir, resolve_weights_variant def _merge_lora_adapter(base_model_id, lora_path, token=None): @@ -56,7 +56,7 @@ def cmd_convert(args): return 1 source_model_id = merged_dir - platform = resolve_platform(getattr(args, "platform", "auto")) + platform = resolve_weights_variant(getattr(args, "weights", "general")) output_dir = args.output_dir or str( get_bundle_dir(args.model_id, bits=args.bits, platform=platform) ) diff --git a/python/cactus/cli/download.py b/python/cactus/cli/download.py index 67beb9261..14192e34b 100644 --- a/python/cactus/cli/download.py +++ b/python/cactus/cli/download.py @@ -3,7 +3,7 @@ from .common import ( BLUE, GREEN, - SUPPORTED_PLATFORMS, + SUPPORTED_WEIGHTS_VARIANTS, print_color, weights_root, ) @@ -21,23 +21,15 @@ def get_bundle_dir(model_id: str, *, bits: int = 4, platform: str | None = None) return weights_root() / f"{get_model_dir_name(model_id)}-{variant_suffix(bits, platform)}" -def _host_platform() -> str | None: - """Best-fit platform for the current host: apple on macOS, else generic CPU.""" - import platform as _platform - - if _platform.system() == "Darwin" and "apple" in SUPPORTED_PLATFORMS: - return "apple" - return None - - -def resolve_platform(choice: str) -> str | None: - if choice == "auto": - return _host_platform() - if choice == "cpu": +def resolve_weights_variant(choice: str | None) -> str | None: + """Map the --weights choice to the bundle variant suffix: general (the + default on every platform) selects the portable bundle (no suffix).""" + if choice in (None, "general"): return None - if choice in SUPPORTED_PLATFORMS: + if choice in SUPPORTED_WEIGHTS_VARIANTS: return choice - raise ValueError(f"unknown platform {choice!r}; supported: auto, cpu, {', '.join(SUPPORTED_PLATFORMS)}") + raise ValueError(f"unknown weights variant {choice!r}; " + f"supported: general, {', '.join(SUPPORTED_WEIGHTS_VARIANTS)}") def ensure_model(model_id: str) -> Path: diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index 0bcf8ab81..483067597 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -238,15 +238,15 @@ def ensure_runnable_bundle(model_id, *, bits=4, platform=None, token=None, def prepare_bundle(args, *, model_id=None, transpile=None, prebuilt=True, output_dir=None, fail_prefix="Model setup failed"): - """Resolve the platform from args and return a runnable bundle, with uniform + """Resolve the weights variant from args and return a runnable bundle, with uniform error handling shared by every model command. Returns the bundle Path, or None (after printing the error) on failure.""" - from .download import resolve_platform + from .download import resolve_weights_variant try: return ensure_runnable_bundle( args.model_id if model_id is None else model_id, bits=getattr(args, "bits", 4), - platform=resolve_platform(getattr(args, "platform", "auto")), + platform=resolve_weights_variant(getattr(args, "weights", "general")), token=getattr(args, "token", None), reconvert=getattr(args, "reconvert", False), prebuilt=prebuilt, diff --git a/python/cactus/cli/upload.py b/python/cactus/cli/upload.py index b0277b737..2a9e9486f 100644 --- a/python/cactus/cli/upload.py +++ b/python/cactus/cli/upload.py @@ -4,7 +4,7 @@ from pathlib import Path from .common import BLUE, GREEN, PROJECT_ROOT, print_color -from .download import get_model_dir_name, resolve_platform +from .download import get_model_dir_name, resolve_weights_variant from .model import prepare_bundle from .utils import suggested_cq_repo, variant_suffix @@ -15,7 +15,7 @@ def cmd_upload(args): if not token: raise SystemExit("missing HuggingFace token: pass --token or set HF_TOKEN") - platform = resolve_platform(args.platform) + platform = resolve_weights_variant(getattr(args, "weights", "general")) repo_id = suggested_cq_repo(args.model_id) stem = f"{get_model_dir_name(args.model_id)}-{variant_suffix(args.bits, platform)}" archive_name = f"{stem}.zip" diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py index 818e8275b..79ad255dd 100644 --- a/python/tests/test_bindings.py +++ b/python/tests/test_bindings.py @@ -336,14 +336,14 @@ def test_run_rejects_bare_name(self): with pytest.raises(SystemExit): self.parser.parse_args(["run", "whisper-base", "--prompt", "hi"]) - def test_run_platform_flag(self): - args = self.parser.parse_args(["run", "Foo/Bar", "--platform", "apple"]) - assert args.platform == "apple" + def test_run_weights_flag(self): + args = self.parser.parse_args(["run", "Foo/Bar", "--weights", "apple"]) + assert args.weights == "apple" assert args.bits == 4 - def test_download_platform_flag(self): - args = self.parser.parse_args(["download", "Foo/Bar", "--platform", "cpu", "--bits", "2"]) - assert args.platform == "cpu" + def test_download_weights_flag(self): + args = self.parser.parse_args(["download", "Foo/Bar", "--weights", "general", "--bits", "2"]) + assert args.weights == "general" assert args.bits == 2 diff --git a/python/tests/test_cli_run.py b/python/tests/test_cli_run.py index 5c4c0d7e5..97aa6a59c 100644 --- a/python/tests/test_cli_run.py +++ b/python/tests/test_cli_run.py @@ -39,7 +39,7 @@ def fake_subprocess_run(cmd): no_cloud_tele=False, model_id="org/model", bits=4, - platform="cpu", + weights="general", token=None, reconvert=False, system=None, @@ -52,6 +52,7 @@ def fake_subprocess_run(cmd): result_json=str(result_json), confidence_threshold=None, cloud_timeout_ms=None, + backend="auto", thinking=False, no_cloud_handoff=False, ) diff --git a/python/tests/test_model.py b/python/tests/test_model.py index b6b859ad7..3addbffd7 100644 --- a/python/tests/test_model.py +++ b/python/tests/test_model.py @@ -17,10 +17,10 @@ cactus_transcribe, ) from cactus.cli.model import ensure_bundle -from cactus.cli.download import resolve_platform +from cactus.cli.download import resolve_weights_variant -_PLATFORM = resolve_platform("auto") +_PLATFORM = resolve_weights_variant("general") _ASSETS_DIR = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" _TEST_IMAGE = _ASSETS_DIR / "test_monkey.png" _TEST_AUDIO = _ASSETS_DIR / "test.wav" From 5217de36a136b4e6a5e9ffad63d0a47052acbc23 Mon Sep 17 00:00:00 2001 From: jakmro Date: Sun, 5 Jul 2026 00:39:14 +0200 Subject: [PATCH 28/36] remove compilation warnings --- apple/CMakeLists.txt | 4 ++-- apple/Info.plist | 2 +- apple/build.sh | 8 +++++--- cactus-engine/src/npu_ane.h | 1 - cactus-engine/src/npu_ane.mm | 2 +- cactus-kernels/cmake/embed_msl.cmake | 10 +++------- cactus-kernels/src/metal_backend.mm | 4 +--- 7 files changed, 13 insertions(+), 18 deletions(-) diff --git a/apple/CMakeLists.txt b/apple/CMakeLists.txt index 2fabddd8c..cdd95297d 100644 --- a/apple/CMakeLists.txt +++ b/apple/CMakeLists.txt @@ -1,8 +1,8 @@ cmake_minimum_required(VERSION 3.10) project(CactusApple LANGUAGES CXX) -set(CMAKE_OSX_DEPLOYMENT_TARGET "13.0" CACHE STRING "") -set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "13.0" CACHE STRING "") +set(CMAKE_OSX_DEPLOYMENT_TARGET "13.3" CACHE STRING "") +set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "16.4" CACHE STRING "") set(CMAKE_BUILD_TYPE Release CACHE STRING "") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) diff --git a/apple/Info.plist b/apple/Info.plist index 91f8988f6..401ef4214 100644 --- a/apple/Info.plist +++ b/apple/Info.plist @@ -19,6 +19,6 @@ CFBundleVersion 1.0.0 MinimumOSVersion - 13.0 + 13.3 diff --git a/apple/build.sh b/apple/build.sh index 69b2ebe75..f71ee5c5a 100755 --- a/apple/build.sh +++ b/apple/build.sh @@ -1,4 +1,6 @@ #!/bin/bash +IOS_MIN_VERSION=16.4 +MACOS_MIN_VERSION=13.3 set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -59,7 +61,7 @@ function build_static_library() { cmake -DCMAKE_SYSTEM_NAME=iOS \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=$IOS_MIN_VERSION \ -DCMAKE_OSX_SYSROOT="$IOS_SDK_PATH" \ -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \ -DBUILD_SHARED_LIBS=OFF \ @@ -86,7 +88,7 @@ function build_static_library() { cmake -DCMAKE_SYSTEM_NAME=iOS \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=$IOS_MIN_VERSION \ -DCMAKE_OSX_SYSROOT="$IOS_SIM_SDK_PATH" \ -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \ -DBUILD_SHARED_LIBS=OFF \ @@ -110,7 +112,7 @@ function build_framework() { -DCMAKE_SYSTEM_NAME=$1 \ -DCMAKE_OSX_ARCHITECTURES="$2" \ -DCMAKE_OSX_SYSROOT=$3 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=$([ "$1" = "iOS" ] && echo "$IOS_MIN_VERSION" || echo "$MACOS_MIN_VERSION") \ -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \ -DBUILD_SHARED_LIBS=ON \ -DCACTUS_CURL_ROOT="$CACTUS_CURL_ROOT" \ diff --git a/cactus-engine/src/npu_ane.h b/cactus-engine/src/npu_ane.h index 7594960bf..07b678687 100644 --- a/cactus-engine/src/npu_ane.h +++ b/cactus-engine/src/npu_ane.h @@ -14,7 +14,6 @@ namespace npu { #if CACTUS_HAS_ANE - class ANEEncoder : public NPUEncoder { public: ANEEncoder(); diff --git a/cactus-engine/src/npu_ane.mm b/cactus-engine/src/npu_ane.mm index 99638e2e0..3bcf2b51c 100644 --- a/cactus-engine/src/npu_ane.mm +++ b/cactus-engine/src/npu_ane.mm @@ -1016,7 +1016,7 @@ - (size_t)predictMultiInputWithDict:(NSMutableDictionary create_encoder() { - return std::make_unique(); + return std::make_unique(); } bool is_npu_available() { diff --git a/cactus-kernels/cmake/embed_msl.cmake b/cactus-kernels/cmake/embed_msl.cmake index 12cab3bb1..500fde994 100644 --- a/cactus-kernels/cmake/embed_msl.cmake +++ b/cactus-kernels/cmake/embed_msl.cmake @@ -1,8 +1,4 @@ -file(READ "${IN}" SRC) -if(SRC MATCHES "\\)${DELIM}\"") - message(FATAL_ERROR "embed_msl: source contains the raw-string delimiter )${DELIM}\" - choose another DELIM") -endif() +file(READ "${IN}" SRC_HEX HEX) +string(REGEX REPLACE "(..)" "0x\\1," SRC_BYTES "${SRC_HEX}") file(WRITE "${OUT}" - "static const char* kCactusMSL = R\"${DELIM}(\n" - "${SRC}\n" - ")${DELIM}\";\n") + "static const char kCactusMSL[] = {${SRC_BYTES}0x00};\n") diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index d18f31712..df94c73b8 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -282,9 +282,7 @@ inline bool tg_mem_ok(size_t bytes) { size_t wf16_budget() { static const size_t cap = [] { if (const char* e = std::getenv("CACTUS_WF16_CAP_MB")) return (size_t)atoll(e) << 20; - if (@available(macOS 10.12, iOS 16.0, *)) - return (size_t)([ctx().dev recommendedMaxWorkingSetSize] / 4); - return (size_t)([NSProcessInfo processInfo].physicalMemory / 4); + return (size_t)([ctx().dev recommendedMaxWorkingSetSize] / 4); }(); return cap; } From 73e47e187ca80470f7b7cd1a05056b2e1750e5ea Mon Sep 17 00:00:00 2001 From: jakmro Date: Sun, 5 Jul 2026 01:07:56 +0200 Subject: [PATCH 29/36] fix --- cactus-graph/src/metal_plan.cpp | 2 +- cactus-graph/src/ops_cache.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index 07eeb8417..80ad50f18 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -1302,7 +1302,7 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, bool has_new = (c.u1 & 0x10000u) != 0; size_t ng = (hd + 31) / 32; size_t win = anchor.params.window_size; - bool sliding = win > 0 && win < nodes[c.b2]->params.max_cache_seq_len; + bool sliding = win > 0; if (has_new && !sliding && clen >= mx) { size_t ceiling = nodes[c.b2]->params.max_cache_seq_len; if (!cactus_kv_cache_grow(kc, clen + 1, ceiling) diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index 29464ebfc..f0ecd9d78 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -176,7 +176,7 @@ void compute_kv_cache_state_node( bool sliding = window > 0 && window < ceiling; size_t max_seq; if (sliding) max_seq = std::min(ceiling, window + node.params.cache_sink_size + 1); - else if (num_slots > 1) max_seq = ceiling; + else if (num_slots > 1 || window > 0) max_seq = ceiling; else max_seq = std::min(ceiling, kInitialCacheEntries); size_t kv_heads = node.params.num_kv_heads; size_t hdim = node.params.head_dim; From bcb5d0bfee16854a487b51da2b6fff06c92db741 Mon Sep 17 00:00:00 2001 From: jakmro Date: Mon, 6 Jul 2026 03:59:01 +0200 Subject: [PATCH 30/36] optimize gemma 4 26b --- cactus-graph/cactus_graph.h | 2 + cactus-graph/src/execute.cpp | 65 +++- cactus-graph/src/metal_plan.cpp | 224 ++++++++++++- cactus-kernels/metal_backend.h | 17 + cactus-kernels/src/cactus_kernels.metal | 375 ++++++++++++++++++++++ cactus-kernels/src/metal_backend.mm | 289 ++++++++++++++++- cactus-kernels/src/metal_backend_stub.cpp | 14 + 7 files changed, 979 insertions(+), 7 deletions(-) diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index fb6dfe27d..4d08a1ef9 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -791,6 +791,8 @@ class CactusGraph { void build_metal_retype_plan(); void invalidate_metal_state(); std::unordered_map metal_plans_; + std::unordered_map> metal_plan_banned_; + uint64_t metal_plan_sig_ = 0; std::vector metal_retype_plan_; bool metal_retype_built_ = false; bool metal_retype_disabled_ = false; diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index f2125f6af..a7872776e 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -865,6 +865,53 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n if (cols == 0) return false; return cactus_metal_encode_softmax_rows(out.get_data(), in.get_data(), out.total_size / cols, cols); } + case OpType::TOPK: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || out.precision != Precision::FP32) return false; + if (in.shape.size() != 2 || node.params.top_k == 0 || node.params.top_k > 16) return false; + return cactus_metal_encode_topk_rows(out.get_data(), in.get_data(), + in.shape[0], in.shape[1], node.params.top_k); + } + case OpType::MOE_LAYER: { + const size_t num_experts = node.params.num_experts; + const size_t top_k = node.params.num_experts_per_tok; + if (!node.params.moe_gated || num_experts == 0 || top_k == 0 || top_k > 16) return false; + if (node.input_ids.size() != 3 + 3 * num_experts) return false; + const auto& hidden = get_input(node, 0, nodes, map); + const auto& probs = get_input(node, 1, nodes, map); + const auto& topk = get_input(node, 2, nodes, map); + if (!fp16(hidden) || !fp16(out) || !fp16(probs)) return false; + if (topk.precision != Precision::FP32) return false; + if (hidden.shape.size() != 2 || probs.shape.size() != 2 || probs.shape[1] != num_experts) return false; + uint32_t act; + switch (node.params.activation) { + case Activation::SILU: act = 0u; break; + case Activation::GELU: act = 1u; break; + default: return false; + } + const auto& w1_0 = get_input(node, 3, nodes, map); + if (!PrecisionTraits::is_cq(w1_0.precision) || w1_0.group_size == 0) return false; + CactusQuantMatrix key = w1_0.to_cq_matrix(); + if (!cactus_metal_moe_cq4_ready(&key)) { + std::vector w1s(num_experts), w3s(num_experts), w2s(num_experts); + for (size_t e = 0; e < num_experts; ++e) { + const auto& b1 = get_input(node, 3 + e, nodes, map); + const auto& b3 = get_input(node, 3 + num_experts + e, nodes, map); + const auto& b2 = get_input(node, 3 + 2 * num_experts + e, nodes, map); + if (!PrecisionTraits::is_cq(b1.precision) || b1.group_size == 0 || + !PrecisionTraits::is_cq(b3.precision) || b3.group_size == 0 || + !PrecisionTraits::is_cq(b2.precision) || b2.group_size == 0) return false; + w1s[e] = b1.to_cq_matrix(); + w3s[e] = b3.to_cq_matrix(); + w2s[e] = b2.to_cq_matrix(); + } + if (!cactus_metal_moe_cq4_build(w1s.data(), w3s.data(), w2s.data(), (uint32_t)num_experts)) + return false; + } + return cactus_metal_encode_moe_gated_cq4(out.get_data(), hidden.get_data(), probs.get_data(), + topk.get_data(), &key, (uint32_t)num_experts, (uint32_t)top_k, (uint32_t)hidden.shape[0], + act, node.params.normalize_routing ? 1u : 0u, node.params.epsilon, node.params.scalar); + } case OpType::LAYERNORM: { if (node.input_ids.size() < 2 || out.shape.empty()) return false; const auto& in = get_input(node, 0, nodes, map); @@ -1153,7 +1200,8 @@ struct MetalFusePlan; MetalFusePlan* cactus_metal_plan_build(const std::vector>& nodes, const std::unordered_map& map, const std::unordered_set& pinned_ids, - const std::vector& retype); + const std::vector& retype, + const std::unordered_set* banned = nullptr); void cactus_metal_plan_free(MetalFusePlan* p); void cactus_metal_plan_disable(MetalFusePlan* p); bool cactus_metal_plan_fold(MetalFusePlan* p, const std::vector>& nodes); @@ -1249,6 +1297,7 @@ void CactusGraph::execute(const std::string& profile_file) { }; bool trace_execution = get_env_int("CACTUS_TRACE_EXECUTE", 0) != 0; + static const size_t flush_cadence = (size_t)get_env_int("CACTUS_FLUSH_CADENCE", 48); bool trace_nan = get_env_int("CACTUS_TRACE_NAN", 0) != 0; bool need_debug = !profile_file.empty(); if (!need_debug) { @@ -1342,9 +1391,13 @@ void CactusGraph::execute(const std::string& profile_file) { static const std::vector no_retype; const std::vector& rt = (!metal_retype_disabled_ && metal_retype_plan_.size() == n) ? metal_retype_plan_ : no_retype; - it = metal_plans_.emplace(sig, cactus_metal_plan_build(nodes_, node_index_map_, pinned, rt)).first; + auto bit = metal_plan_banned_.find(sig); + const std::unordered_set* banned = + bit == metal_plan_banned_.end() ? nullptr : &bit->second; + it = metal_plans_.emplace(sig, cactus_metal_plan_build(nodes_, node_index_map_, pinned, rt, banned)).first; } fplan = it->second; + metal_plan_sig_ = sig; } const uint8_t* rplan = (metal_mode && !need_debug && !metal_retype_disabled_ && metal_retype_plan_.size() == n) ? metal_retype_plan_.data() : nullptr; @@ -1569,13 +1622,15 @@ void CactusGraph::execute(const std::string& profile_file) { if (cactus_metal_plan_encode(fplan, fact, nodes_, node_index_map_)) { metal_live[i] = 1; for (size_t r : release_after[i]) metal_release(r); - if (++since_flush >= 48) { cactus_metal_session_flush(); since_flush = 0; } + if (++since_flush >= flush_cadence) { cactus_metal_session_flush(); since_flush = 0; } maybe_recycle(); continue; } - cactus_metal_plan_disable(fplan); metal_guard.armed = false; metal_abort_cleanup(); + metal_plan_banned_[metal_plan_sig_].insert(i); + metal_plans_.erase(metal_plan_sig_); + cactus_metal_plan_free(fplan); execute(profile_file); return; } @@ -1613,7 +1668,7 @@ void CactusGraph::execute(const std::string& profile_file) { } if (ot == OpType::PERSISTENT) populated_node_ids_.insert(node->id); for (size_t r : release_after[i]) metal_release(r); - if (encoded && ++since_flush >= 48) { cactus_metal_session_flush(); since_flush = 0; } + if (encoded && ++since_flush >= flush_cadence) { cactus_metal_session_flush(); since_flush = 0; } if (encoded) maybe_recycle(); } metal_guard.armed = false; diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index 80ad50f18..194b8fab5 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -62,7 +62,8 @@ MetalFusePlan* cactus_metal_plan_build( const std::vector>& nodes, const std::unordered_map& map, const std::unordered_set& pinned_ids, - const std::vector& retype) { + const std::vector& retype, + const std::unordered_set* banned) { const size_t n = nodes.size(); auto plan = new MetalFusePlan(); plan->action.assign(n, -1); @@ -223,6 +224,7 @@ MetalFusePlan* cactus_metal_plan_build( for (auto& sp : c.sc) if (sp) { cactus_metal_free_shared(sp); sp = nullptr; } }; auto add_cluster = [&](MetalCluster c, size_t anchor, const std::vector& cover) -> bool { + if (banned && banned->count(anchor)) { release_scratch(c); return false; } for (size_t v : cover) if (v != anchor && pinned[v]) { release_scratch(c); return false; } int32_t cid = (int32_t)plan->clusters.size(); plan->clusters.push_back(c); @@ -255,6 +257,7 @@ MetalFusePlan* cactus_metal_plan_build( long kcache = idxof(nd.input_ids[3]); long vcache = idxof(nd.input_ids[4]); if (qk < 0 || vk < 0 || kcache < 0 || vcache < 0) continue; + if (nodes[(size_t)kcache]->params.num_kv_heads != 1) continue; std::vector cover; long q_rope = up_in(i, 0); @@ -1006,6 +1009,190 @@ MetalFusePlan* cactus_metal_plan_build( cover.push_back((size_t)mm); add_cluster(c, i, cover); } + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd0 = *nodes[i]; + if (nd0.op_type != OpType::ADD || nd0.input_ids.size() != 2) continue; + if (nd0.output_buffer.precision != Precision::FP16 || retyped(i)) continue; + const auto& osh = nd0.output_buffer.shape; + if (osh.size() < 2) continue; + size_t D = osh.back(); + if (D == 0 || (D % 2) != 0) continue; + size_t H = nd0.output_buffer.total_size / D; + if (H == 0 || H * D != nd0.output_buffer.total_size) continue; + long mA = up_in(i, 0), mB = up_in(i, 1); + if (mA < 0 || mB < 0 || mA == mB) continue; + auto is_mul = [&](long j) { + return nodes[(size_t)j]->op_type == OpType::MULTIPLY + && nodes[(size_t)j]->input_ids.size() == 2 + && plan->action[(size_t)j] == -1 && !pinned[(size_t)j] && !retyped((size_t)j) + && nodes[(size_t)j]->output_buffer.precision == Precision::FP16 + && nodes[(size_t)j]->output_buffer.total_size == H * D; + }; + if (!is_mul(mA) || !is_mul(mB)) continue; + auto split_mul = [&](long m, long* big, long* small) -> bool { + long i0 = up_in((size_t)m, 0), i1 = up_in((size_t)m, 1); + if (i0 < 0 || i1 < 0) return false; + size_t t0 = nodes[(size_t)i0]->output_buffer.total_size; + size_t t1 = nodes[(size_t)i1]->output_buffer.total_size; + if (t0 == H * D && t1 == D) { *big = i0; *small = i1; return true; } + if (t1 == H * D && t0 == D) { *big = i1; *small = i0; return true; } + return false; + }; + long xA = -1, cA = -1, xB = -1, sB = -1; + if (!split_mul(mA, &xA, &cA) || !split_mul(mB, &xB, &sB)) continue; + long mulCos = -1, mulSin = -1, xi = -1, ci = -1, si = -1, cat = -1; + if (nodes[(size_t)xB]->op_type == OpType::CAT) { + mulCos = mA; mulSin = mB; xi = xA; ci = cA; si = sB; cat = xB; + } else if (nodes[(size_t)xA]->op_type == OpType::CAT) { + mulCos = mB; mulSin = mA; xi = xB; ci = sB; si = cA; cat = xA; + } else continue; + GraphNode& cn = *nodes[(size_t)cat]; + if (cn.input_ids.size() != 2 || plan->action[(size_t)cat] != -1 + || pinned[(size_t)cat] || retyped((size_t)cat)) continue; + if (cn.output_buffer.total_size != H * D) continue; + size_t cat_ax = (size_t)cn.params.axis; + if (cat_ax != cn.output_buffer.shape.size() - 1) continue; + long neg = up_in((size_t)cat, 0); + long lo = up_in((size_t)cat, 1); + if (neg < 0 || lo < 0) continue; + GraphNode& ng = *nodes[(size_t)neg]; + if (ng.op_type != OpType::SCALAR_MULTIPLY || ng.params.scalar != -1.0f + || plan->action[(size_t)neg] != -1 || pinned[(size_t)neg] || retyped((size_t)neg)) continue; + long hi = up_in((size_t)neg, 0); + if (hi < 0) continue; + GraphNode& hs = *nodes[(size_t)hi]; + GraphNode& ls = *nodes[(size_t)lo]; + if (hs.op_type != OpType::SLICE || ls.op_type != OpType::SLICE) continue; + if (plan->action[(size_t)hi] != -1 || plan->action[(size_t)lo] != -1 + || pinned[(size_t)hi] || pinned[(size_t)lo]) continue; + if (hs.params.slice_start != D / 2 || ls.params.slice_start != 0) continue; + size_t hlen = hs.params.slice_length ? hs.params.slice_length : D / 2; + size_t llen = ls.params.slice_length ? ls.params.slice_length : D / 2; + if (hlen != D / 2 || llen != D / 2) continue; + if ((size_t)hs.params.axis != hs.output_buffer.shape.size() - 1 + || (size_t)ls.params.axis != ls.output_buffer.shape.size() - 1) continue; + long xh = up_in((size_t)hi, 0), xl = up_in((size_t)lo, 0); + if (xh != xi || xl != xi) continue; + if (nodes[(size_t)xi]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)ci]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)si]->output_buffer.precision != Precision::FP16) continue; + if (!sole_use(mulCos, {i}) || !sole_use(mulSin, {i}) + || !sole_use(cat, {(size_t)mulSin}) || !sole_use(neg, {(size_t)cat}) + || !sole_use(hi, {(size_t)neg}) || !sole_use(lo, {(size_t)cat})) continue; + MetalCluster c; + c.rule = 14; + c.a0 = (size_t)xi; c.a1 = (size_t)ci; c.a2 = (size_t)si; + c.u0 = (uint32_t)H; c.u1 = (uint32_t)D; + c.b4 = i; + std::vector cover{(size_t)mulCos, (size_t)mulSin, (size_t)cat, + (size_t)neg, (size_t)hi, (size_t)lo}; + collect_chain((size_t)idxof(nd0.input_ids[0]), mulCos, cover); + collect_chain((size_t)idxof(nd0.input_ids[1]), mulSin, cover); + GraphNode& xn = *nodes[(size_t)xi]; + if (D <= 1024 && xn.op_type == OpType::RMS_NORM && xn.input_ids.size() >= 2 + && plan->action[(size_t)xi] == -1 && !pinned[(size_t)xi] && !retyped((size_t)xi) + && sole_use(xi, {(size_t)mulCos, (size_t)hi, (size_t)lo})) { + long rsrc = up_in((size_t)xi, 0); + long rw = up_in((size_t)xi, 1); + if (rw >= 0) rw = deep_f16_source(rw); + if (rsrc >= 0 && rw >= 0 + && nodes[(size_t)rsrc]->output_buffer.precision == Precision::FP16 + && nodes[(size_t)rsrc]->output_buffer.total_size == H * D + && nodes[(size_t)rw]->output_buffer.precision == Precision::FP16 + && nodes[(size_t)rw]->output_buffer.total_size == D) { + c.rule = 17; + c.a0 = (size_t)rsrc; c.a3 = (size_t)rw; + c.f0 = xn.params.epsilon; + cover.push_back((size_t)xi); + } + } + add_cluster(c, i, cover); + } + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd0 = *nodes[i]; + if (nd0.op_type != OpType::SCALAR_MULTIPLY || nd0.input_ids.empty()) continue; + if (nd0.output_buffer.precision != Precision::FP16 || retyped(i)) continue; + const auto& osh = nd0.output_buffer.shape; + if (osh.empty()) continue; + size_t D = osh.back(); + size_t rows = nd0.output_buffer.total_size / D; + if (D < 32 || rows == 0 || rows * D != nd0.output_buffer.total_size) continue; + long rn = up_in(i, 0); + if (rn < 0 || nodes[(size_t)rn]->op_type != OpType::RMS_NORM + || nodes[(size_t)rn]->input_ids.size() < 2 + || plan->action[(size_t)rn] != -1 || pinned[(size_t)rn] || retyped((size_t)rn) + || nodes[(size_t)rn]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)rn]->output_buffer.total_size != rows * D) continue; + if (!sole_use(rn, {i})) continue; + long src = up_in((size_t)rn, 0); + long w = up_in((size_t)rn, 1); + if (w >= 0) w = deep_f16_source(w); + if (src < 0 || w < 0 + || nodes[(size_t)src]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)w]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)w]->output_buffer.total_size != D) continue; + MetalCluster c; + c.rule = 15; + c.a0 = (size_t)src; c.a1 = (size_t)w; + c.u0 = (uint32_t)rows; c.u1 = (uint32_t)D; + c.f0 = nodes[(size_t)rn]->params.epsilon; + c.f1 = nd0.params.scalar; + c.b4 = i; + std::vector cover{(size_t)rn}; + collect_chain((size_t)idxof(nd0.input_ids[0]), rn, cover); + add_cluster(c, i, cover); + } + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd0 = *nodes[i]; + if (nd0.op_type != OpType::TOPK || nd0.input_ids.empty()) continue; + if (nd0.output_buffer.precision != Precision::FP32) continue; + size_t k = nd0.params.top_k; + if (k == 0 || k > 16) continue; + long sm = up_in(i, 0); + if (sm < 0 || nodes[(size_t)sm]->op_type != OpType::SCALAR_MULTIPLY + || nodes[(size_t)sm]->input_ids.empty() + || plan->action[(size_t)sm] != -1 || pinned[(size_t)sm] || retyped((size_t)sm) + || nodes[(size_t)sm]->output_buffer.precision != Precision::FP16) continue; + const auto& ssh = nodes[(size_t)sm]->output_buffer.shape; + if (ssh.size() != 2) continue; + size_t rows = ssh[0], E = ssh[1]; + if (rows == 0 || E == 0 || E > 4096) continue; + if (nd0.output_buffer.total_size != 2 * rows * k) continue; + long sfm = -1; + for (size_t cc : cons[(size_t)sm]) { + if (cc == i) continue; + const GraphNode& cn = *nodes[cc]; + if (cn.op_type == OpType::SOFTMAX && plan->action[cc] == -1 + && !pinned[cc] && !retyped(cc) + && cn.output_buffer.precision == Precision::FP16 + && cn.output_buffer.total_size == rows * E) { + if (sfm < 0) sfm = (long)cc; + else { sfm = -1; break; } + } else if (!is_alias_op(cn.op_type)) { sfm = -1; break; } + } + if (sfm < 0 || !sole_use(sm, {i, (size_t)sfm})) continue; + if ((size_t)sfm > i) continue; + bool ordered = true; + for (size_t cc : cons[(size_t)sfm]) + if (cc < i) { ordered = false; break; } + if (!ordered) continue; + long lg = up_in((size_t)sm, 0); + if (lg < 0 || nodes[(size_t)lg]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)lg]->output_buffer.total_size != rows * E) continue; + MetalCluster c; + c.rule = 16; + c.a0 = (size_t)lg; c.a1 = (size_t)sfm; + c.u0 = (uint32_t)rows; c.u1 = (uint32_t)E; + c.b0 = k; + c.f0 = nodes[(size_t)sm]->params.scalar; + c.b4 = i; + std::vector cover{(size_t)sm}; + collect_chain((size_t)idxof(nd0.input_ids[0]), sm, cover); + if (add_cluster(c, i, cover)) plan->action[(size_t)sfm] = -3; + } for (size_t i = 0; i < n; ++i) { if (plan->action[i] != -1) continue; int kind, code; float p0, p1; @@ -1554,6 +1741,41 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, if (!x || !w || !b || !y) return false; return cactus_metal_encode_gemv_bias(y, x, w, b, c.u0, c.u1, (int)c.b0); } + case 14: { + GraphNode& anchor = *nodes[c.b4]; + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* cs = nodes[c.a1]->output_buffer.get_data(); + const void* sn = nodes[c.a2]->output_buffer.get_data(); + void* y = anchor.output_buffer.get_data(); + if (!x || !cs || !sn || !y) return false; + return cactus_metal_encode_rope_pair(y, x, cs, sn, c.u0, c.u1); + } + case 15: { + GraphNode& anchor = *nodes[c.b4]; + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* w = nodes[c.a1]->output_buffer.get_data(); + void* y = anchor.output_buffer.get_data(); + if (!x || !w || !y) return false; + return cactus_metal_encode_rms_norm_scale(y, x, w, c.u0, c.u1, c.f0, c.f1); + } + case 16: { + GraphNode& anchor = *nodes[c.b4]; + const void* lg = nodes[c.a0]->output_buffer.get_data(); + void* probs = nodes[c.a1]->output_buffer.get_data(); + void* tk = anchor.output_buffer.get_data(); + if (!lg || !probs || !tk) return false; + return cactus_metal_encode_softmax_topk(probs, tk, lg, c.u0, c.u1, c.b0, c.f0); + } + case 17: { + GraphNode& anchor = *nodes[c.b4]; + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* w = nodes[c.a3]->output_buffer.get_data(); + const void* cs = nodes[c.a1]->output_buffer.get_data(); + const void* sn = nodes[c.a2]->output_buffer.get_data(); + void* y = anchor.output_buffer.get_data(); + if (!x || !w || !cs || !sn || !y) return false; + return cactus_metal_encode_rope_pair_rms(y, x, w, cs, sn, c.u0, c.u1, c.f0); + } default: return false; } } diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 445cdeb1a..f67c59243 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -66,6 +66,23 @@ bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, bool cactus_metal_encode_swiglu_transform(void* code, const void* gate, const void* up, const CactusQuantMatrix* W, float scale); bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W); +bool cactus_metal_encode_rope_pair(void* out, const void* x, const void* c, const void* s, + uint32_t H, uint32_t D); +bool cactus_metal_encode_rope_pair_rms(void* out, const void* x, const void* w, + const void* c, const void* s, + uint32_t H, uint32_t D, float eps); +bool cactus_metal_encode_rms_norm_scale(void* out, const void* in, const void* weight, + size_t rows, size_t dim, float eps, float oscale); +bool cactus_metal_encode_softmax_topk(void* probs, void* topk, const void* in, + size_t rows, size_t cols, size_t k, float scale); +bool cactus_metal_encode_topk_rows(void* out, const void* in, size_t rows, size_t cols, size_t k); +bool cactus_metal_moe_cq4_ready(const CactusQuantMatrix* w1_0); +bool cactus_metal_moe_cq4_build(const CactusQuantMatrix* w1s, const CactusQuantMatrix* w3s, + const CactusQuantMatrix* w2s, uint32_t num_experts); +bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void* probs, + const void* topk, const CactusQuantMatrix* w1_0, + uint32_t num_experts, uint32_t top_k, uint32_t tokens, + uint32_t act, uint32_t normalize, float eps, float scaling); bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* code, const CactusQuantMatrix* W); bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W, float scale); diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 03488338b..20b57c2dc 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -3260,3 +3260,378 @@ kernel void argmax_logits(device const half* logits [[buffer(0)]], } if (t == 0u) { out3[0] = sb[0]; out3[1] = ss[0]; out3[2] = (float)si[0]; } } + +kernel void topk_rows_f16(device const half* in [[buffer(0)]], + device float* out [[buffer(1)]], + constant uint& F [[buffer(2)]], + constant uint& k [[buffer(3)]], + constant uint& B [[buffer(4)]], + uint row [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]]) { + device const half* x = in + (size_t)row*F; + device float* idx_out = out + (size_t)row*k; + device float* val_out = out + (size_t)B*k + (size_t)row*k; + for (uint j = 0u; j < k; ++j) { + float best = -INFINITY; uint bi = 0xFFFFFFFFu; + for (uint i = lane; i < F; i += 32u) { + bool taken = false; + for (uint p = 0u; p < j; ++p) if ((uint)idx_out[p] == i) taken = true; + float v = (float)x[i]; + if (!taken && (v > best || (v == best && i < bi))) { best = v; bi = i; } + } + float gb = simd_max(best); + uint cand = (best == gb) ? bi : 0xFFFFFFFFu; + uint gi = simd_min(cand); + if (lane == 0u) { idx_out[j] = (float)gi; val_out[j] = gb; } + simdgroup_barrier(mem_flags::mem_device); + } +} + +kernel void cq4_moe_transform( + device const half* x [[buffer(0)]], + device const float* topk [[buffer(1)]], + device const half* recip_c [[buffer(2)]], + device const char* lsign_c [[buffer(3)]], + device const char* rsign_c [[buffer(4)]], + device const uint* perm_c [[buffer(5)]], + device half* code [[buffer(6)]], + constant uint& K [[buffer(7)]], + constant uint& k_valid [[buffer(8)]], + constant uint& xz_stride [[buffer(9)]], + constant uint& xy_stride [[buffer(10)]], + constant uint& topk_k [[buffer(11)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + threadgroup float* zmem [[threadgroup(0)]]) +{ + const uint g = tgp.x, slot = tgp.y, tok = tgp.z; + const uint e = (uint)(topk[(size_t)tok*topk_k + slot] + 0.5f); + device const half* recip = recip_c + (size_t)e*K; + device const char* lsign = lsign_c + (size_t)e*128u; + device const char* rsign = rsign_c + (size_t)e*128u; + device const uint* perm = perm_c + (size_t)e*128u; + device const half* xs = x + (size_t)tok*xz_stride + (size_t)slot*xy_stride; + uint b = g*128u + lane*4u; + uint k = lane*4u; + float xv[4]; + #pragma clang loop unroll(full) + for (uint j = 0u; j < 4u; ++j) { + float v = (b+j < k_valid) ? (float)xs[b+j] : 0.0f; + xv[j] = v*(float)recip[b+j]*(float)lsign[k+j]; + } + float x0 = xv[0], x1 = xv[1], x2 = xv[2], x3 = xv[3]; + cq4_hada128(x0, x1, x2, x3, lane); + zmem[k+0] = x0*(float)rsign[k+0]; zmem[k+1] = x1*(float)rsign[k+1]; + zmem[k+2] = x2*(float)rsign[k+2]; zmem[k+3] = x3*(float)rsign[k+3]; + threadgroup_barrier(mem_flags::mem_threadgroup); + device half* co = code + ((size_t)tok*topk_k + slot)*K + g*128u; + co[k+0] = (half)zmem[perm[k+0]]; co[k+1] = (half)zmem[perm[k+1]]; + co[k+2] = (half)zmem[perm[k+2]]; co[k+3] = (half)zmem[perm[k+3]]; +} + +kernel void cq4_moe_gemv_up( + device const half* code1 [[buffer(0)]], + device const half* code3 [[buffer(1)]], + device const float* topk [[buffer(2)]], + device const uchar* pk1 [[buffer(3)]], + device const half* nm1 [[buffer(4)]], + device const half* cb1 [[buffer(5)]], + device const uchar* pk3 [[buffer(6)]], + device const half* nm3 [[buffer(7)]], + device const half* cb3 [[buffer(8)]], + device half* y [[buffer(9)]], + constant uint& K [[buffer(10)]], + constant uint& N [[buffer(11)]], + constant uint& act [[buffer(12)]], + constant uint& topk_k [[buffer(13)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]]) +{ + const uint slot = tgp.y, tok = tgp.z; + const uint slotg = tok*topk_k + slot; + const uint e = (uint)(topk[(size_t)tok*topk_k + slot] + 0.5f); + const uint ng = K/128u, pgb = 64u; + threadgroup float c1[16], c3[16]; + if (tl < 16u) { c1[tl] = (float)cb1[(size_t)e*16u+tl]; c3[tl] = (float)cb3[(size_t)e*16u+tl]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + uint n0 = (tgp.x*8u + sgid)*4u; + if (n0 >= N) return; + float gv[4] = {0,0,0,0}, uv[4] = {0,0,0,0}; + for (int m = 0; m < 2; ++m) { + device const uchar* pb = (m==0) ? pk1 + (size_t)e*N*ng*pgb : pk3 + (size_t)e*N*ng*pgb; + device const half* nb = (m==0) ? nm1 + (size_t)e*N*ng : nm3 + (size_t)e*N*ng; + device const half* cd = (m==0) ? code1 + (size_t)slotg*K : code3 + (size_t)slotg*K; + threadgroup float* cb = (m==0) ? c1 : c3; + float a0=0, a1=0, a2=0, a3=0; + for (uint base = lane*16u; base < K; base += 512u) { + uint gg = base/128u, off = base - gg*128u; + device const half4* cp = (device const half4*)(cd + gg*128u + off); + half4 c0=cp[0], c1v=cp[1], c2v=cp[2], c3v=cp[3]; + device const ushort4* q0 = (device const ushort4*)(pb + ((size_t)(n0+0u)*ng+gg)*pgb + off/2u); + device const ushort4* q1 = (device const ushort4*)(pb + ((size_t)(n0+1u)*ng+gg)*pgb + off/2u); + device const ushort4* q2 = (device const ushort4*)(pb + ((size_t)(n0+2u)*ng+gg)*pgb + off/2u); + device const ushort4* q3 = (device const ushort4*)(pb + ((size_t)(n0+3u)*ng+gg)*pgb + off/2u); + a0 += (float)nb[(size_t)(n0+0u)*ng+gg]*cq4_dot64(c0, c1v, c2v, c3v, q0[0], cb); + a1 += (float)nb[(size_t)(n0+1u)*ng+gg]*cq4_dot64(c0, c1v, c2v, c3v, q1[0], cb); + a2 += (float)nb[(size_t)(n0+2u)*ng+gg]*cq4_dot64(c0, c1v, c2v, c3v, q2[0], cb); + a3 += (float)nb[(size_t)(n0+3u)*ng+gg]*cq4_dot64(c0, c1v, c2v, c3v, q3[0], cb); + } + a0 = simd_sum(a0); a1 = simd_sum(a1); a2 = simd_sum(a2); a3 = simd_sum(a3); + if (m == 0) { gv[0]=a0; gv[1]=a1; gv[2]=a2; gv[3]=a3; } + else { uv[0]=a0; uv[1]=a1; uv[2]=a2; uv[3]=a3; } + } + if (lane == 0u) { + device half* ys = y + (size_t)slotg*N; + for (uint r = 0u; r < 4u && n0+r < N; ++r) { + float gh = (float)(half)gv[r]; + float av = (act == 1u) ? gelu_tanh(gh) : gh/(1.0f + precise::exp(-gh)); + ys[n0+r] = (half)((float)(half)av*(float)(half)uv[r]); + } + } +} + +kernel void cq4_moe_transform2( + device const half* x [[buffer(0)]], + device const float* topk [[buffer(1)]], + device const half* recip_c1 [[buffer(2)]], + device const char* lsign_c1 [[buffer(3)]], + device const char* rsign_c1 [[buffer(4)]], + device const uint* perm_c1 [[buffer(5)]], + device const half* recip_c3 [[buffer(6)]], + device const char* lsign_c3 [[buffer(7)]], + device const char* rsign_c3 [[buffer(8)]], + device const uint* perm_c3 [[buffer(9)]], + device half* code1 [[buffer(10)]], + device half* code3 [[buffer(11)]], + constant uint& K [[buffer(12)]], + constant uint& topk_k [[buffer(13)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + threadgroup float* zmem [[threadgroup(0)]]) +{ + const uint g = tgp.x, tok = tgp.z; + const bool w3 = tgp.y >= topk_k; + const uint slot = w3 ? tgp.y - topk_k : tgp.y; + const uint e = (uint)(topk[(size_t)tok*topk_k + slot] + 0.5f); + device const half* recip = (w3 ? recip_c3 : recip_c1) + (size_t)e*K; + device const char* lsign = (w3 ? lsign_c3 : lsign_c1) + (size_t)e*128u; + device const char* rsign = (w3 ? rsign_c3 : rsign_c1) + (size_t)e*128u; + device const uint* perm = (w3 ? perm_c3 : perm_c1 ) + (size_t)e*128u; + device const half* xs = x + (size_t)tok*K; + uint b = g*128u + lane*4u; + uint k = lane*4u; + float x0=(float)xs[b+0]*(float)recip[b+0]*(float)lsign[k+0]; + float x1=(float)xs[b+1]*(float)recip[b+1]*(float)lsign[k+1]; + float x2=(float)xs[b+2]*(float)recip[b+2]*(float)lsign[k+2]; + float x3=(float)xs[b+3]*(float)recip[b+3]*(float)lsign[k+3]; + cq4_hada128(x0, x1, x2, x3, lane); + zmem[k+0]=x0*(float)rsign[k+0]; zmem[k+1]=x1*(float)rsign[k+1]; + zmem[k+2]=x2*(float)rsign[k+2]; zmem[k+3]=x3*(float)rsign[k+3]; + threadgroup_barrier(mem_flags::mem_threadgroup); + device half* co = (w3 ? code3 : code1) + ((size_t)tok*topk_k + slot)*K + g*128u; + co[k+0]=(half)zmem[perm[k+0]]; co[k+1]=(half)zmem[perm[k+1]]; + co[k+2]=(half)zmem[perm[k+2]]; co[k+3]=(half)zmem[perm[k+3]]; +} + +kernel void cq4_moe_gemv_down_acc( + device const half* code2 [[buffer(0)]], + device const float* topk [[buffer(1)]], + device const half* probs [[buffer(2)]], + device const uchar* pk2 [[buffer(3)]], + device const half* nm2 [[buffer(4)]], + device const half* cb2 [[buffer(5)]], + device half* out [[buffer(6)]], + constant uint& K [[buffer(7)]], + constant uint& N [[buffer(8)]], + constant uint& topk_k [[buffer(9)]], + constant uint& normalize [[buffer(10)]], + constant float& eps [[buffer(11)]], + constant float& scaling [[buffer(12)]], + constant uint& E [[buffer(13)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]], + threadgroup float* cbs [[threadgroup(0)]]) +{ + const uint tok = tgp.z; + device const float* tk = topk + (size_t)tok*topk_k; + device const half* pr = probs + (size_t)tok*E; + uint order[16]; + for (uint j = 0u; j < topk_k; ++j) order[j] = j; + for (uint a = 1u; a < topk_k; ++a) { + uint v = order[a]; + uint ev = (uint)(tk[v] + 0.5f); + uint b = a; + while (b > 0u && (uint)(tk[order[b-1u]] + 0.5f) > ev) { order[b] = order[b-1u]; --b; } + order[b] = v; + } + if (tl < topk_k*16u) { + uint s = tl >> 4, ci = tl & 15u; + uint e = (uint)(tk[s] + 0.5f); + cbs[tl] = (float)cb2[(size_t)e*16u + ci]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + float denom = 1.0f; + if (normalize != 0u) { + float s = 0.0f; + for (uint j = 0u; j < topk_k; ++j) s += (float)pr[(uint)(tk[j] + 0.5f)]; + denom = s + eps; + } + const uint ng = K/128u, pgb = 64u; + uint n0 = (tgp.x*8u + sgid)*2u; + if (n0 >= N) return; + half res[2] = {0.0h, 0.0h}; + for (uint j = 0u; j < topk_k; ++j) { + uint slot = order[j]; + uint e = (uint)(tk[slot] + 0.5f); + float p = (float)pr[e]; + if (p <= 0.0f) continue; + float w = p/denom*scaling; + device const uchar* pbase = pk2 + (size_t)e*N*ng*pgb; + device const half* nbase = nm2 + (size_t)e*N*ng; + device const half* cd = code2 + ((size_t)tok*topk_k + slot)*K; + threadgroup const float* cb = cbs + (size_t)slot*16u; + float acc[2] = {0, 0}; + for (uint base = lane*16u; base < K; base += 512u) { + uint gg = base/128u, off = base - gg*128u; + device const half4* cp = (device const half4*)(cd + gg*128u + off); + half4 c0=cp[0], c1=cp[1], c2=cp[2], c3=cp[3]; + #pragma clang loop unroll(full) + for (uint r = 0u; r < 2u; ++r) { + device const ushort4* q = (device const ushort4*)(pbase + ((size_t)(n0+r)*ng+gg)*pgb + off/2u); + acc[r] += (float)nbase[(size_t)(n0+r)*ng+gg]*cq4_dot64(c0, c1, c2, c3, q[0], cb); + } + } + #pragma clang loop unroll(full) + for (uint r = 0u; r < 2u; ++r) { + float a = simd_sum(acc[r]); + res[r] = (half)((float)res[r] + (float)(half)a*w); + } + } + if (lane == 0u) { + device half* ys = out + (size_t)tok*N; + for (uint r = 0u; r < 2u && n0+r < N; ++r) ys[n0+r] = res[r]; + } +} + +kernel void rope_pair_f16( + device const half* x [[buffer(0)]], + device const half* c [[buffer(1)]], + device const half* s [[buffer(2)]], + device half* y [[buffer(3)]], + constant uint& H [[buffer(4)]], + constant uint& D [[buffer(5)]], + uint2 g [[thread_position_in_grid]]) +{ + uint d = g.x, h = g.y; + if (d >= D || h >= H) return; + size_t idx = (size_t)h*D + d; + half xv = x[idx]; + half rh = (d < D/2u) ? (half)(-x[idx + D/2u]) : x[idx - D/2u]; + y[idx] = xv*c[d] + rh*s[d]; +} + +kernel void rms_norm_scale_f16( + device const half* in [[buffer(0)]], + device const half* w [[buffer(1)]], + device half* y [[buffer(2)]], + constant uint& D [[buffer(3)]], + constant float& eps [[buffer(4)]], + constant float& oscale [[buffer(5)]], + uint row [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* red [[threadgroup(0)]]) +{ + device const half* x = in + (size_t)row*D; + device half* o = y + (size_t)row*D; + float ss = 0; + for (uint i = t; i < D; i += T) { float v = (float)x[i]; ss += v*v; } + red[t] = ss; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint s = T/2; s > 0; s >>= 1) { + if (t < s) red[t] += red[t+s]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + float inv = rsqrt(red[0]/(float)D + eps); + for (uint i = t; i < D; i += T) { + half y0 = (half)((float)x[i]*inv*(float)w[i]); + o[i] = (half)((float)y0*oscale); + } +} + +kernel void softmax_topk_f16( + device const half* in [[buffer(0)]], + device half* probs [[buffer(1)]], + device float* tk [[buffer(2)]], + constant uint& E [[buffer(3)]], + constant uint& k [[buffer(4)]], + constant uint& B [[buffer(5)]], + constant float& scale [[buffer(6)]], + uint row [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]]) +{ + device const half* x = in + (size_t)row*E; + device half* pr = probs + (size_t)row*E; + device float* idx_out = tk + (size_t)row*k; + device float* val_out = tk + (size_t)B*k + (size_t)row*k; + float mx = -INFINITY; + for (uint i = lane; i < E; i += 32u) mx = max(mx, (float)(half)((float)x[i]*scale)); + mx = simd_max(mx); + float sum = 0; + for (uint i = lane; i < E; i += 32u) sum += exp((float)(half)((float)x[i]*scale) - mx); + sum = simd_sum(sum); + float inv = 1.0f/sum; + for (uint i = lane; i < E; i += 32u) pr[i] = (half)(exp((float)(half)((float)x[i]*scale) - mx)*inv); + for (uint j = 0u; j < k; ++j) { + float best = -INFINITY; uint bi = 0xFFFFFFFFu; + for (uint i = lane; i < E; i += 32u) { + bool taken = false; + for (uint p = 0u; p < j; ++p) if ((uint)idx_out[p] == i) taken = true; + float v = (float)(half)((float)x[i]*scale); + if (!taken && (v > best || (v == best && i < bi))) { best = v; bi = i; } + } + float gb = simd_max(best); + uint cand = (best == gb) ? bi : 0xFFFFFFFFu; + uint gi = simd_min(cand); + if (lane == 0u) { idx_out[j] = (float)gi; val_out[j] = gb; } + simdgroup_barrier(mem_flags::mem_device); + } +} + +kernel void rope_pair_rms_f16( + device const half* x [[buffer(0)]], + device const half* w [[buffer(1)]], + device const half* c [[buffer(2)]], + device const half* s [[buffer(3)]], + device half* y [[buffer(4)]], + constant uint& D [[buffer(5)]], + constant float& eps [[buffer(6)]], + uint h [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* red [[threadgroup(0)]], + threadgroup half* xn [[threadgroup(1)]]) +{ + device const half* xr = x + (size_t)h*D; + float ss = 0; + for (uint i = t; i < D; i += T) { float v = (float)xr[i]; ss += v*v; } + red[t] = ss; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint st = T/2; st > 0; st >>= 1) { + if (t < st) red[t] += red[t+st]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + float inv = rsqrt(red[0]/(float)D + eps); + for (uint i = t; i < D; i += T) xn[i] = (half)((float)xr[i]*inv*(float)w[i]); + threadgroup_barrier(mem_flags::mem_threadgroup); + device half* yr = y + (size_t)h*D; + for (uint i = t; i < D; i += T) { + half xv = xn[i]; + half rh = (i < D/2u) ? (half)(-xn[i + D/2u]) : xn[i - D/2u]; + yr[i] = xv*c[i] + rh*s[i]; + } +} diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index df94c73b8..359ee0b49 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -24,6 +24,8 @@ #include #include "cactus_kernels_msl.h" +#include "threading.h" +#include namespace { @@ -61,6 +63,9 @@ id psoAttnFlash=nil, psoRmsSimd=nil, psoScatterRows=nil; id psoTranspose2d=nil, psoBcastRows=nil, psoRmsAddSimd=nil; id psoConvCacheAppend=nil, psoRelPosBias=nil, psoGemvBias=nil; + id psoTopkRows=nil, psoMoeT=nil, psoMoeUp=nil; + id psoMoeT2=nil, psoMoeDownAcc=nil, psoRopePair=nil; + id psoRmsScale=nil, psoSoftmaxTopk=nil, psoRopePairRms=nil; id dummy=nil; bool ok = false; @@ -121,6 +126,11 @@ psoBcastRows=pso("bcast_binary_rows_f16"); psoRmsAddSimd=pso("rms_norm_add_simd_f16"); psoConvCacheAppend=pso("conv_cache_append_f16"); psoRelPosBias=pso("rel_pos_bias_f16"); psoGemvBias=pso("gemv_bias_f16"); + psoTopkRows=pso("topk_rows_f16"); psoMoeT=pso("cq4_moe_transform"); + psoMoeUp=pso("cq4_moe_gemv_up"); psoMoeT2=pso("cq4_moe_transform2"); + psoMoeDownAcc=pso("cq4_moe_gemv_down_acc"); psoRopePair=pso("rope_pair_f16"); + psoRmsScale=pso("rms_norm_scale_f16"); psoSoftmaxTopk=pso("softmax_topk_f16"); + psoRopePairRms=pso("rope_pair_rms_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotW&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoArgmax&&psoGather; @@ -164,6 +174,14 @@ static uint64_t resident_key(const CactusQuantMatrix* W) { std::unordered_map g_resident_emb; std::mutex g_resident_mu; +struct MoESet4 { + id pk=nil, nm=nil, rc=nil, ls=nil, rs=nil, pm=nil, cb=nil; + uint32_t K=0, N=0, ng=0; +}; +struct MoECat4 { MoESet4 w1, w3, w2; bool ok=false; }; +std::unordered_map g_moe_cat; +size_t g_moe_bytes = 0; + void deinterleave_4row(const CactusQuantMatrix* W, uint8_t* packed_nat, __fp16* norms_nat) { const uint32_t N=W->N, ng=W->num_groups, gs=W->group_size, pgb=(gs*4u+7u)/8u, NB=N/4; @@ -284,7 +302,10 @@ size_t wf16_budget() { if (const char* e = std::getenv("CACTUS_WF16_CAP_MB")) return (size_t)atoll(e) << 20; return (size_t)([ctx().dev recommendedMaxWorkingSetSize] / 4); }(); - return cap; + const size_t floor_cap = 64u << 20; + if (g_moe_bytes >= cap) return floor_cap; + size_t left = cap - g_moe_bytes; + return left < floor_cap ? floor_cap : left; } void wf16_evict_for(size_t need, const ResW* keep) { @@ -457,6 +478,8 @@ void cactus_metal_invalidate_host_wraps() { g_dequant_cache.clear(); g_mps_mats.clear(); g_wf16_bytes = 0; + g_moe_cat.clear(); + g_moe_bytes = 0; } void cactus_metal_session_flush() { @@ -943,6 +966,270 @@ bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { return true; } +namespace { +void moe_release_source(const CactusQuantMatrix* W) { + const uint32_t pgb = (W->group_size*W->bits + 7u)/8u; + struct Span { const void* p; size_t len; }; + Span spans[7] = { + { W->packed_indices, (size_t)W->N*W->num_groups*pgb }, + { W->norms, (size_t)W->N*W->num_groups*2 }, + { W->input_scale_recip, (size_t)W->K*2 }, + { W->left_signs, (size_t)W->group_size }, + { W->right_signs, (size_t)W->group_size }, + { W->permutation, (size_t)W->group_size*4 }, + { W->codebook, (size_t)(1u<bits)*2 }, + }; + uintptr_t lo = UINTPTR_MAX, hi = 0; + for (const auto& s : spans) { + if (!s.p) continue; + uintptr_t a = (uintptr_t)s.p; + if (a < lo) lo = a; + if (a + s.len > hi) hi = a + s.len; + } + if (hi <= lo) return; + const uintptr_t pg = 16384; + uintptr_t start = (lo + pg - 1) & ~(pg - 1), end = hi & ~(pg - 1); + if (end > start) madvise((void*)start, end - start, MADV_DONTNEED); +} + +bool moe_build_set4(MoESet4& S, const CactusQuantMatrix* Ws, uint32_t count) { + const uint32_t K=Ws[0].K, N=Ws[0].N, gs=Ws[0].group_size, ng=Ws[0].num_groups; + if (gs != 128u || Ws[0].bits != 4u || (N % 4u) != 0u || (K % 128u) != 0u) return false; + for (uint32_t e = 0; e < count; ++e) { + const CactusQuantMatrix* W = &Ws[e]; + if (!quant_fast_eligible(W) || W->K != K || W->N != N || W->num_groups != ng) return false; + } + const uint32_t pgb = 64u; + const size_t pk_stride = (size_t)N*ng*pgb, nm_stride = (size_t)N*ng; + S.K=K; S.N=N; S.ng=ng; + S.pk=[ctx().dev newBufferWithLength:pk_stride*count options:MTLResourceStorageModeShared]; + S.nm=[ctx().dev newBufferWithLength:nm_stride*count*2 options:MTLResourceStorageModeShared]; + S.rc=[ctx().dev newBufferWithLength:(size_t)K*count*2 options:MTLResourceStorageModeShared]; + S.ls=[ctx().dev newBufferWithLength:(size_t)128*count options:MTLResourceStorageModeShared]; + S.rs=[ctx().dev newBufferWithLength:(size_t)128*count options:MTLResourceStorageModeShared]; + S.pm=[ctx().dev newBufferWithLength:(size_t)128*count*4 options:MTLResourceStorageModeShared]; + S.cb=[ctx().dev newBufferWithLength:(size_t)16*count*2 options:MTLResourceStorageModeShared]; + if (!S.pk||!S.nm||!S.rc||!S.ls||!S.rs||!S.pm||!S.cb) return false; + CactusThreading::parallel_for(count, CactusThreading::ParallelConfig(1, 1), + [&](size_t start, size_t end) { + for (size_t e = start; e < end; ++e) { + const CactusQuantMatrix* W = &Ws[e]; + deinterleave_4row(W, (uint8_t*)S.pk.contents + pk_stride*e, + (__fp16*)S.nm.contents + nm_stride*e); + std::memcpy((__fp16*)S.rc.contents + (size_t)K*e, W->input_scale_recip, (size_t)K*2); + std::memcpy((char*)S.ls.contents + 128*e, W->left_signs, 128); + std::memcpy((char*)S.rs.contents + 128*e, W->right_signs, 128); + std::memcpy((uint32_t*)S.pm.contents + 128*e, W->permutation, 128*4); + std::memcpy((__fp16*)S.cb.contents + 16*e, W->codebook, 16*2); + moe_release_source(W); + } + }); + return true; +} +} + +bool cactus_metal_moe_cq4_ready(const CactusQuantMatrix* w1_0) { + std::lock_guard lk(g_resident_mu); + auto it = g_moe_cat.find(resident_key(w1_0)); + return it != g_moe_cat.end() && it->second.ok; +} + +bool cactus_metal_moe_cq4_build(const CactusQuantMatrix* w1s, const CactusQuantMatrix* w3s, + const CactusQuantMatrix* w2s, uint32_t num_experts) { + if (!ctx().ok || !ctx().psoMoeT || !ctx().psoMoeT2 || !ctx().psoMoeUp || !ctx().psoMoeDownAcc) + return false; + const uint64_t key = resident_key(&w1s[0]); + { + std::lock_guard lk(g_resident_mu); + auto it = g_moe_cat.find(key); + if (it != g_moe_cat.end()) return it->second.ok; + } + MoECat4 cat; + cat.ok = moe_build_set4(cat.w1, w1s, num_experts) + && moe_build_set4(cat.w3, w3s, num_experts) + && moe_build_set4(cat.w2, w2s, num_experts) + && cat.w1.K == cat.w3.K && cat.w1.N == cat.w3.N && cat.w2.K >= cat.w1.N; + if (!cat.ok) cat = MoECat4{}; + std::lock_guard lk(g_resident_mu); + auto ins = g_moe_cat.emplace(key, cat); + if (ins.second && cat.ok) { + for (const MoESet4* st : {&cat.w1, &cat.w3, &cat.w2}) + for (id b : {st->pk, st->nm, st->rc, st->ls, st->rs, st->pm, st->cb}) + if (b) g_moe_bytes += (size_t)b.length; + } + return ins.first->second.ok; +} + +bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void* probs, + const void* topk, const CactusQuantMatrix* w1_0, + uint32_t num_experts, uint32_t top_k, uint32_t tokens, + uint32_t act, uint32_t normalize, float eps, float scaling) { + if (!ctx().ok || !ctx().psoMoeT || top_k == 0 || top_k > 16 || tokens == 0) return false; + g_resident_mu.lock(); + auto it = g_moe_cat.find(resident_key(w1_0)); + bool hit = it != g_moe_cat.end() && it->second.ok; + g_resident_mu.unlock(); + if (!hit) return false; + MoECat4& C = it->second; + const uint32_t K1=C.w1.K, N1=C.w1.N, K2=C.w2.K, N2=C.w2.N; + const size_t slots = (size_t)tokens*top_k; + ensureEncoder(); + + id code1 = recycled(slots*K1*2); + id code3 = recycled(slots*K1*2); + id gu = recycled(slots*N1*2); + id code2 = recycled(slots*K2*2); + + { + [g_enc setComputePipelineState:ctx().psoMoeT2]; + setBufAt(hidden, (size_t)tokens*K1*2, 0); + setBufAt(topk, slots*sizeof(float), 1); + [g_enc setBuffer:C.w1.rc offset:0 atIndex:2]; + [g_enc setBuffer:C.w1.ls offset:0 atIndex:3]; + [g_enc setBuffer:C.w1.rs offset:0 atIndex:4]; + [g_enc setBuffer:C.w1.pm offset:0 atIndex:5]; + [g_enc setBuffer:C.w3.rc offset:0 atIndex:6]; + [g_enc setBuffer:C.w3.ls offset:0 atIndex:7]; + [g_enc setBuffer:C.w3.rs offset:0 atIndex:8]; + [g_enc setBuffer:C.w3.pm offset:0 atIndex:9]; + [g_enc setBuffer:code1 offset:0 atIndex:10]; + [g_enc setBuffer:code3 offset:0 atIndex:11]; + uint32_t Kc=K1, tk=top_k; + [g_enc setBytes:&Kc length:4 atIndex:12]; [g_enc setBytes:&tk length:4 atIndex:13]; + [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(K1/128u, 2u*top_k, tokens) + threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + } + { + [g_enc setComputePipelineState:ctx().psoMoeUp]; + [g_enc setBuffer:code1 offset:0 atIndex:0]; [g_enc setBuffer:code3 offset:0 atIndex:1]; + setBufAt(topk, slots*sizeof(float), 2); + [g_enc setBuffer:C.w1.pk offset:0 atIndex:3]; [g_enc setBuffer:C.w1.nm offset:0 atIndex:4]; [g_enc setBuffer:C.w1.cb offset:0 atIndex:5]; + [g_enc setBuffer:C.w3.pk offset:0 atIndex:6]; [g_enc setBuffer:C.w3.nm offset:0 atIndex:7]; [g_enc setBuffer:C.w3.cb offset:0 atIndex:8]; + [g_enc setBuffer:gu offset:0 atIndex:9]; + uint32_t Kc=K1, Nc=N1, a=act, tk=top_k; + [g_enc setBytes:&Kc length:4 atIndex:10]; [g_enc setBytes:&Nc length:4 atIndex:11]; + [g_enc setBytes:&a length:4 atIndex:12]; [g_enc setBytes:&tk length:4 atIndex:13]; + [g_enc dispatchThreadgroups:MTLSizeMake((N1+31u)/32u, top_k, tokens) + threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } + { + [g_enc setComputePipelineState:ctx().psoMoeT]; + [g_enc setBuffer:gu offset:0 atIndex:0]; + setBufAt(topk, slots*sizeof(float), 1); + [g_enc setBuffer:C.w2.rc offset:0 atIndex:2]; + [g_enc setBuffer:C.w2.ls offset:0 atIndex:3]; + [g_enc setBuffer:C.w2.rs offset:0 atIndex:4]; + [g_enc setBuffer:C.w2.pm offset:0 atIndex:5]; + [g_enc setBuffer:code2 offset:0 atIndex:6]; + uint32_t Kc=K2, kv=N1, xz=top_k*N1, xy=N1, tk=top_k; + [g_enc setBytes:&Kc length:4 atIndex:7]; [g_enc setBytes:&kv length:4 atIndex:8]; + [g_enc setBytes:&xz length:4 atIndex:9]; [g_enc setBytes:&xy length:4 atIndex:10]; + [g_enc setBytes:&tk length:4 atIndex:11]; + [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(K2/128u, top_k, tokens) + threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + } + { + [g_enc setComputePipelineState:ctx().psoMoeDownAcc]; + [g_enc setBuffer:code2 offset:0 atIndex:0]; + setBufAt(topk, slots*sizeof(float), 1); + setBufAt(probs, (size_t)tokens*num_experts*2, 2); + [g_enc setBuffer:C.w2.pk offset:0 atIndex:3]; [g_enc setBuffer:C.w2.nm offset:0 atIndex:4]; [g_enc setBuffer:C.w2.cb offset:0 atIndex:5]; + setBufAt(out, (size_t)tokens*N2*2, 6); + uint32_t Kc=K2, Nc=N2, tk=top_k, nz=normalize, Ec=num_experts; + [g_enc setBytes:&Kc length:4 atIndex:7]; [g_enc setBytes:&Nc length:4 atIndex:8]; + [g_enc setBytes:&tk length:4 atIndex:9]; [g_enc setBytes:&nz length:4 atIndex:10]; + [g_enc setBytes:&eps length:4 atIndex:11]; [g_enc setBytes:&scaling length:4 atIndex:12]; + [g_enc setBytes:&Ec length:4 atIndex:13]; + [g_enc setThreadgroupMemoryLength:top_k*16*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((N2+15u)/16u, 1, tokens) + threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } + return true; +} + +bool cactus_metal_encode_rms_norm_scale(void* out, const void* in, const void* weight, + size_t rows, size_t dim, float eps, float oscale) { + if (!ctx().ok || !ctx().psoRmsScale || rows == 0 || dim == 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoRmsScale]; + setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(out, rows*dim*2, 2); + uint32_t d=(uint32_t)dim; float e=eps, sc=oscale; + [g_enc setBytes:&d length:4 atIndex:3]; [g_enc setBytes:&e length:4 atIndex:4]; + [g_enc setBytes:&sc length:4 atIndex:5]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} + +bool cactus_metal_encode_softmax_topk(void* probs, void* topk, const void* in, + size_t rows, size_t cols, size_t k, float scale) { + if (!ctx().ok || !ctx().psoSoftmaxTopk || k == 0 || k > 16 || rows == 0 || cols == 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoSoftmaxTopk]; + setBufAt(in, rows*cols*2, 0); + setBufAt(probs, rows*cols*2, 1); + setBufAt(topk, rows*k*2*sizeof(float), 2); + uint32_t E=(uint32_t)cols, kc=(uint32_t)k, B=(uint32_t)rows; float sc=scale; + [g_enc setBytes:&E length:4 atIndex:3]; + [g_enc setBytes:&kc length:4 atIndex:4]; + [g_enc setBytes:&B length:4 atIndex:5]; + [g_enc setBytes:&sc length:4 atIndex:6]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + return true; +} + +bool cactus_metal_encode_rope_pair(void* out, const void* x, const void* c, const void* s, + uint32_t H, uint32_t D) { + if (!ctx().ok || !ctx().psoRopePair || H == 0 || D == 0 || (D % 2) != 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoRopePair]; + setBufAt(x, (size_t)H*D*2, 0); + setBufAt(c, (size_t)D*2, 1); + setBufAt(s, (size_t)D*2, 2); + setBufAt(out, (size_t)H*D*2, 3); + [g_enc setBytes:&H length:4 atIndex:4]; + [g_enc setBytes:&D length:4 atIndex:5]; + [g_enc dispatchThreads:MTLSizeMake(D, H, 1) threadsPerThreadgroup:MTLSizeMake(64, 4, 1)]; + return true; +} + +bool cactus_metal_encode_rope_pair_rms(void* out, const void* x, const void* w, + const void* c, const void* s, + uint32_t H, uint32_t D, float eps) { + if (!ctx().ok || !ctx().psoRopePairRms || H == 0 || D == 0 || (D % 2) != 0 || D > 1024) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoRopePairRms]; + setBufAt(x, (size_t)H*D*2, 0); + setBufAt(w, (size_t)D*2, 1); + setBufAt(c, (size_t)D*2, 2); + setBufAt(s, (size_t)D*2, 3); + setBufAt(out, (size_t)H*D*2, 4); + float e = eps; + [g_enc setBytes:&D length:4 atIndex:5]; + [g_enc setBytes:&e length:4 atIndex:6]; + uint32_t T = D < 256 ? D : 256; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:(size_t)D*2 atIndex:1]; + [g_enc dispatchThreadgroups:MTLSizeMake(H,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + return true; +} + +bool cactus_metal_encode_topk_rows(void* out, const void* in, size_t rows, size_t cols, size_t k) { + if (!ctx().ok || !ctx().psoTopkRows || k == 0 || k > 16 || rows == 0 || cols == 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoTopkRows]; + setBufAt(in, rows*cols*2, 0); + setBufAt(out, rows*k*2*sizeof(float), 1); + uint32_t F=(uint32_t)cols, kc=(uint32_t)k, B=(uint32_t)rows; + [g_enc setBytes:&F length:4 atIndex:2]; + [g_enc setBytes:&kc length:4 atIndex:3]; + [g_enc setBytes:&B length:4 atIndex:4]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + return true; +} + bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M) { if (!ctx().ok || !W) return false; if (M == 1 && (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 0f7daa4f9..a8e38902b 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -113,4 +113,18 @@ bool cactus_metal_encode_rel_pos_bias(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, int, float) { return false; } bool cactus_metal_encode_gemv_bias(void*, const void*, const void*, const void*, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_rope_pair(void*, const void*, const void*, const void*, + uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_rope_pair_rms(void*, const void*, const void*, const void*, + const void*, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_rms_norm_scale(void*, const void*, const void*, + size_t, size_t, float, float) { return false; } +bool cactus_metal_encode_softmax_topk(void*, void*, const void*, + size_t, size_t, size_t, float) { return false; } +bool cactus_metal_encode_topk_rows(void*, const void*, size_t, size_t, size_t) { return false; } +bool cactus_metal_moe_cq4_ready(const CactusQuantMatrix*) { return false; } +bool cactus_metal_moe_cq4_build(const CactusQuantMatrix*, const CactusQuantMatrix*, + const CactusQuantMatrix*, uint32_t) { return false; } +bool cactus_metal_encode_moe_gated_cq4(void*, const void*, const void*, const void*, const CactusQuantMatrix*, + uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, float) { return false; } #endif From e76850dc10e0e08f6cb52a99dd5dd51a75e700ad Mon Sep 17 00:00:00 2001 From: jakmro Date: Mon, 6 Jul 2026 04:27:27 +0200 Subject: [PATCH 31/36] more optim --- cactus-graph/src/metal_plan.cpp | 48 +++++++++++++++++++++++ cactus-kernels/metal_backend.h | 3 ++ cactus-kernels/src/cactus_kernels.metal | 33 ++++++++++++++++ cactus-kernels/src/metal_backend.mm | 20 +++++++++- cactus-kernels/src/metal_backend_stub.cpp | 2 + 5 files changed, 105 insertions(+), 1 deletion(-) diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index 194b8fab5..bf467f0e2 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -695,6 +695,44 @@ MetalFusePlan* cactus_metal_plan_build( if (nd.op_type == OpType::ADD_CLIPPED && nd.input_ids.size() == 2) { long n0 = up_in(i, 0), n1 = up_in(i, 1); + auto rms2_side = [&](long nrm, long* src, long* w) -> bool { + if (nrm < 0 || nodes[(size_t)nrm]->op_type != OpType::RMS_NORM + || nodes[(size_t)nrm]->input_ids.size() < 2 + || plan->action[(size_t)nrm] != -1 || pinned[(size_t)nrm] || retyped((size_t)nrm) + || nodes[(size_t)nrm]->output_buffer.precision != Precision::FP16 + || !sole_use(nrm, {i})) return false; + long s0 = up_in((size_t)nrm, 0); + long w0 = up_in((size_t)nrm, 1); + if (w0 >= 0) w0 = deep_f16_source(w0); + if (s0 < 0 || w0 < 0 + || nodes[(size_t)s0]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)w0]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)s0]->output_buffer.shape.empty() + || nodes[(size_t)s0]->output_buffer.total_size != + nodes[(size_t)s0]->output_buffer.shape.back()) return false; + *src = s0; *w = w0; + return true; + }; + size_t D2 = nd.output_buffer.shape.empty() ? 0 : nd.output_buffer.shape.back(); + long srcA = -1, wA = -1, srcB = -1, wB = -1; + if (n0 != n1 && D2 > 0 && nd.output_buffer.total_size == D2 + && nd.output_buffer.precision == Precision::FP16 && !retyped(i) + && rms2_side(n0, &srcA, &wA) && rms2_side(n1, &srcB, &wB) + && nodes[(size_t)srcA]->output_buffer.total_size == D2 + && nodes[(size_t)srcB]->output_buffer.total_size == D2 + && nodes[(size_t)wA]->output_buffer.total_size == D2 + && nodes[(size_t)wB]->output_buffer.total_size == D2) { + MetalCluster c; + c.rule = 18; + c.a0 = (size_t)srcA; c.a1 = (size_t)wA; + c.a2 = (size_t)srcB; c.a3 = (size_t)wB; + c.u1 = (uint32_t)D2; + c.f0 = nodes[(size_t)n0]->params.epsilon; + c.f1 = nodes[(size_t)n1]->params.epsilon; + c.b4 = i; + std::vector cover{(size_t)n0, (size_t)n1}; + if (add_cluster(c, i, cover)) continue; + } long norm = -1, res = -1; if (n0 >= 0 && nodes[(size_t)n0]->op_type == OpType::RMS_NORM) { norm = n0; res = n1; } else if (n1 >= 0 && nodes[(size_t)n1]->op_type == OpType::RMS_NORM) { norm = n1; res = n0; } @@ -1766,6 +1804,16 @@ bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, if (!lg || !probs || !tk) return false; return cactus_metal_encode_softmax_topk(probs, tk, lg, c.u0, c.u1, c.b0, c.f0); } + case 18: { + GraphNode& anchor = *nodes[c.b4]; + const void* a = nodes[c.a0]->output_buffer.get_data(); + const void* wa = nodes[c.a1]->output_buffer.get_data(); + const void* b = nodes[c.a2]->output_buffer.get_data(); + const void* wb = nodes[c.a3]->output_buffer.get_data(); + void* y = anchor.output_buffer.get_data(); + if (!a || !wa || !b || !wb || !y) return false; + return cactus_metal_encode_rms2_add_clip(y, a, wa, b, wb, c.u1, c.f0, c.f1); + } case 17: { GraphNode& anchor = *nodes[c.b4]; const void* x = nodes[c.a0]->output_buffer.get_data(); diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index f67c59243..84d2760be 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -71,6 +71,9 @@ bool cactus_metal_encode_rope_pair(void* out, const void* x, const void* c, cons bool cactus_metal_encode_rope_pair_rms(void* out, const void* x, const void* w, const void* c, const void* s, uint32_t H, uint32_t D, float eps); +bool cactus_metal_encode_rms2_add_clip(void* out, const void* a, const void* wa, + const void* b, const void* wb, size_t dim, + float eps_a, float eps_b); bool cactus_metal_encode_rms_norm_scale(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps, float oscale); bool cactus_metal_encode_softmax_topk(void* probs, void* topk, const void* in, diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 20b57c2dc..6856a08e8 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -3635,3 +3635,36 @@ kernel void rope_pair_rms_f16( yr[i] = xv*c[i] + rh*s[i]; } } + +kernel void rms2_add_clip_f16( + device const half* a [[buffer(0)]], + device const half* wa [[buffer(1)]], + device const half* b [[buffer(2)]], + device const half* wb [[buffer(3)]], + device half* y [[buffer(4)]], + constant uint& D [[buffer(5)]], + constant float& eps_a [[buffer(6)]], + constant float& eps_b [[buffer(7)]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* red [[threadgroup(0)]]) +{ + float sa = 0, sb = 0; + for (uint i = t; i < D; i += T) { + float va = (float)a[i], vb = (float)b[i]; + sa += va*va; sb += vb*vb; + } + red[t] = sa; red[T+t] = sb; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint s = T/2; s > 0; s >>= 1) { + if (t < s) { red[t] += red[t+s]; red[T+t] += red[T+t+s]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + float inva = rsqrt(red[0]/(float)D + eps_a); + float invb = rsqrt(red[T]/(float)D + eps_b); + for (uint i = t; i < D; i += T) { + half ya = (half)((float)a[i]*inva*(float)wa[i]); + half yb = (half)((float)b[i]*invb*(float)wb[i]); + y[i] = (half)clamp((float)ya + (float)yb, -65500.0f, 65500.0f); + } +} diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index 359ee0b49..b82a80997 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -66,6 +66,7 @@ id psoTopkRows=nil, psoMoeT=nil, psoMoeUp=nil; id psoMoeT2=nil, psoMoeDownAcc=nil, psoRopePair=nil; id psoRmsScale=nil, psoSoftmaxTopk=nil, psoRopePairRms=nil; + id psoRms2AddClip=nil; id dummy=nil; bool ok = false; @@ -130,7 +131,7 @@ psoMoeUp=pso("cq4_moe_gemv_up"); psoMoeT2=pso("cq4_moe_transform2"); psoMoeDownAcc=pso("cq4_moe_gemv_down_acc"); psoRopePair=pso("rope_pair_f16"); psoRmsScale=pso("rms_norm_scale_f16"); psoSoftmaxTopk=pso("softmax_topk_f16"); - psoRopePairRms=pso("rope_pair_rms_f16"); + psoRopePairRms=pso("rope_pair_rms_f16"); psoRms2AddClip=pso("rms2_add_clip_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotW&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoArgmax&&psoGather; @@ -1149,6 +1150,23 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void return true; } +bool cactus_metal_encode_rms2_add_clip(void* out, const void* a, const void* wa, + const void* b, const void* wb, size_t dim, + float eps_a, float eps_b) { + if (!ctx().ok || !ctx().psoRms2AddClip || dim == 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoRms2AddClip]; + setBufAt(a, dim*2, 0); setBufAt(wa, dim*2, 1); + setBufAt(b, dim*2, 2); setBufAt(wb, dim*2, 3); + setBufAt(out, dim*2, 4); + uint32_t d=(uint32_t)dim; float ea=eps_a, eb=eps_b; + [g_enc setBytes:&d length:4 atIndex:5]; [g_enc setBytes:&ea length:4 atIndex:6]; + [g_enc setBytes:&eb length:4 atIndex:7]; + [g_enc setThreadgroupMemoryLength:512*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} + bool cactus_metal_encode_rms_norm_scale(void* out, const void* in, const void* weight, size_t rows, size_t dim, float eps, float oscale) { if (!ctx().ok || !ctx().psoRmsScale || rows == 0 || dim == 0) return false; diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index a8e38902b..0b7237cb1 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -117,6 +117,8 @@ bool cactus_metal_encode_rope_pair(void*, const void*, const void*, const void*, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_rope_pair_rms(void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_rms2_add_clip(void*, const void*, const void*, + const void*, const void*, size_t, float, float) { return false; } bool cactus_metal_encode_rms_norm_scale(void*, const void*, const void*, size_t, size_t, float, float) { return false; } bool cactus_metal_encode_softmax_topk(void*, void*, const void*, From 81d57f4f9242a8804b2ace31a69c90fefe3c1ece Mon Sep 17 00:00:00 2001 From: jakmro Date: Mon, 6 Jul 2026 15:29:37 +0200 Subject: [PATCH 32/36] fixes --- cactus-graph/src/execute.cpp | 18 ++++++++++------- cactus-kernels/metal_backend.h | 3 ++- cactus-kernels/src/blas.cpp | 24 ++++++++++++++++++++--- cactus-kernels/src/cactus_kernels.metal | 7 ++++--- cactus-kernels/src/metal_backend.mm | 9 ++++++--- cactus-kernels/src/metal_backend_stub.cpp | 2 +- 6 files changed, 45 insertions(+), 18 deletions(-) diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index a7872776e..e7081c5c2 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -592,9 +592,12 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n const auto& cin = get_input(node, ii, nodes, map); uint32_t ishape[8], ostride[8]; for (uint32_t d=0; d 0 && cin.shape[0] == 1 && out.shape[0] > 1) ? out.shape[0] : 1; + for (size_t r = 0; r < bcast; ++r) { + if (!cactus_metal_encode_strided_scatter(out.get_data(), cin.get_data(), ishape, ostride, nd, + (uint32_t)cin.total_size, (uint32_t)(axis_off * ostr[axis] + r * ostr[0]), cin.byte_size, out.byte_size)) + return false; + } axis_off += cin.shape[axis]; } return true; @@ -671,11 +674,12 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n int axis = node.params.axis; if (axis < 0) axis += (int)a.shape.size(); if (axis < 0 || (size_t)axis >= a.shape.size() || a.shape.size() != b.shape.size()) return false; - size_t outer = 1, inner = 1; - for (size_t d = 0; d < (size_t)axis; ++d) outer *= a.shape[d]; + size_t a_outer = 1, b_outer = 1, inner = 1; + for (size_t d = 0; d < (size_t)axis; ++d) { a_outer *= a.shape[d]; b_outer *= b.shape[d]; } for (size_t d = (size_t)axis + 1; d < a.shape.size(); ++d) inner *= a.shape[d]; return cactus_metal_encode_concat2(out.get_data(), a.get_data(), b.get_data(), - (uint32_t)outer, (uint32_t)a.shape[(size_t)axis], (uint32_t)b.shape[(size_t)axis], (uint32_t)inner); + (uint32_t)a_outer, (uint32_t)b_outer, + (uint32_t)a.shape[(size_t)axis], (uint32_t)b.shape[(size_t)axis], (uint32_t)inner); } case OpType::GATHER: { const auto& t = get_input(node, 0, nodes, map); @@ -1642,7 +1646,7 @@ void CactusGraph::execute(const std::string& profile_file) { flip(node->output_buffer); for (size_t id : node->input_ids) { auto it = node_index_map_.find(id); - if (it != node_index_map_.end()) flip(nodes_[it->second]->output_buffer); + if (it != node_index_map_.end() && rplan[it->second] != 0) flip(nodes_[it->second]->output_buffer); } } bool force_cpu = (ot == OpType::PRECISION_CAST diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 84d2760be..4bbe96f22 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -139,7 +139,8 @@ bool cactus_metal_encode_reduce_axis(int op, void* out, const void* in, uint32_t bool cactus_metal_encode_cumsum(void* out, const void* in, uint32_t outer, uint32_t axis_size, uint32_t inner, int f32); bool cactus_metal_encode_concat2(void* out, const void* a, const void* b, - uint32_t outer, uint32_t a_axis, uint32_t b_axis, uint32_t inner); + uint32_t a_outer, uint32_t b_outer, + uint32_t a_axis, uint32_t b_axis, uint32_t inner); bool cactus_metal_encode_gather_f32idx(void* out, const void* table, const void* idx, uint32_t rows, uint32_t D, size_t table_bytes); bool cactus_metal_encode_rope_full(void* out, const void* in, uint32_t tokens, uint32_t S, diff --git a/cactus-kernels/src/blas.cpp b/cactus-kernels/src/blas.cpp index 34adab4e9..db58d4e88 100644 --- a/cactus-kernels/src/blas.cpp +++ b/cactus-kernels/src/blas.cpp @@ -331,6 +331,15 @@ void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* outpu size_t axis_size1 = shape1[axis]; size_t axis_size2 = shape2[axis]; + size_t outer_size1 = 1; + size_t outer_size2 = 1; + for (size_t i = 0; i < static_cast(axis); ++i) { + outer_size1 *= shape1[i]; + outer_size2 *= shape2[i]; + } + if (outer_size1 == 0) outer_size1 = 1; + if (outer_size2 == 0) outer_size2 = 1; + size_t input1_stride = axis_size1 * inner_size; size_t input2_stride = axis_size2 * inner_size; size_t output_stride = (axis_size1 + axis_size2) * inner_size; @@ -338,8 +347,8 @@ void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* outpu CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, [&](size_t start, size_t end) { for (size_t outer = start; outer < end; ++outer) { - const __fp16* in1_ptr = input1 + outer * input1_stride; - const __fp16* in2_ptr = input2 + outer * input2_stride; + const __fp16* in1_ptr = input1 + (outer % outer_size1) * input1_stride; + const __fp16* in2_ptr = input2 + (outer % outer_size2) * input2_stride; __fp16* out_ptr = output + outer * output_stride; size_t copy_size1 = axis_size1 * inner_size; @@ -365,6 +374,14 @@ void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_ inner_size *= output_shape[i]; } + std::vector input_outer(num_inputs, 1); + for (size_t input_idx = 0; input_idx < num_inputs; ++input_idx) { + for (size_t i = 0; i < static_cast(axis); ++i) { + input_outer[input_idx] *= input_shapes[input_idx][i]; + } + if (input_outer[input_idx] == 0) input_outer[input_idx] = 1; + } + CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, [&](size_t start, size_t end) { for (size_t outer = start; outer < end; ++outer) { @@ -372,7 +389,8 @@ void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_ size_t offset = 0; for (size_t input_idx = 0; input_idx < num_inputs; ++input_idx) { - const __fp16* in_ptr = inputs[input_idx] + outer * inner_size * input_shapes[input_idx][axis]; + size_t src_outer = outer % input_outer[input_idx]; + const __fp16* in_ptr = inputs[input_idx] + src_outer * inner_size * input_shapes[input_idx][axis]; size_t copy_size = input_shapes[input_idx][axis] * inner_size; std::memcpy(out_ptr + offset, in_ptr, copy_size * sizeof(__fp16)); offset += copy_size; diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 6856a08e8..5079d15a7 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -1932,13 +1932,14 @@ kernel void cumsum_f32(device const float* in [[buffer(0)]], device float* y [[b kernel void concat2_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], device half* y [[buffer(2)]], constant uint& a_axis [[buffer(3)]], constant uint& b_axis [[buffer(4)]], constant uint& inner [[buffer(5)]], - constant uint& n [[buffer(6)]], uint i [[thread_position_in_grid]]) { + constant uint& n [[buffer(6)]], constant uint& a_outer [[buffer(7)]], + constant uint& b_outer [[buffer(8)]], uint i [[thread_position_in_grid]]) { if (i >= n) return; uint in_i = i % inner, rest = i / inner; uint ab = a_axis + b_axis; uint ax = rest % ab, outer = rest / ab; - y[i] = (ax < a_axis) ? a[((size_t)outer*a_axis + ax)*inner + in_i] - : b[((size_t)outer*b_axis + (ax - a_axis))*inner + in_i]; + y[i] = (ax < a_axis) ? a[((size_t)(outer % a_outer)*a_axis + ax)*inner + in_i] + : b[((size_t)(outer % b_outer)*b_axis + (ax - a_axis))*inner + in_i]; } kernel void gather_f32idx_f16(device const half* table [[buffer(0)]], device const float* rows [[buffer(1)]], diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index b82a80997..dc314cdac 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -2000,15 +2000,18 @@ bool cactus_metal_encode_cumsum(void* out, const void* in, uint32_t outer, return true; } bool cactus_metal_encode_concat2(void* out, const void* a, const void* b, - uint32_t outer, uint32_t a_axis, uint32_t b_axis, uint32_t inner) { - if (!ctx().ok || !ctx().psoConcat2) return false; + uint32_t a_outer, uint32_t b_outer, + uint32_t a_axis, uint32_t b_axis, uint32_t inner) { + if (!ctx().ok || !ctx().psoConcat2 || a_outer == 0 || b_outer == 0) return false; ensureEncoder(); + uint32_t outer = a_outer > b_outer ? a_outer : b_outer; uint32_t n = outer * (a_axis + b_axis) * inner; [g_enc setComputePipelineState:ctx().psoConcat2]; - setBufAt(a, (size_t)outer*a_axis*inner*2, 0); setBufAt(b, (size_t)outer*b_axis*inner*2, 1); + setBufAt(a, (size_t)a_outer*a_axis*inner*2, 0); setBufAt(b, (size_t)b_outer*b_axis*inner*2, 1); setBufAt(out, (size_t)n*2, 2); [g_enc setBytes:&a_axis length:4 atIndex:3]; [g_enc setBytes:&b_axis length:4 atIndex:4]; [g_enc setBytes:&inner length:4 atIndex:5]; [g_enc setBytes:&n length:4 atIndex:6]; + [g_enc setBytes:&a_outer length:4 atIndex:7]; [g_enc setBytes:&b_outer length:4 atIndex:8]; [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 0b7237cb1..766b212ec 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -75,7 +75,7 @@ bool cactus_metal_encode_attention_f16(void*, const void*, const void*, const vo float, uint32_t, uint32_t, uint32_t, float, uint32_t) { return false; } bool cactus_metal_encode_reduce_axis(int, void*, const void*, uint32_t, uint32_t, uint32_t, int) { return false; } bool cactus_metal_encode_cumsum(void*, const void*, uint32_t, uint32_t, uint32_t, int) { return false; } -bool cactus_metal_encode_concat2(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_concat2(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_gather_f32idx(void*, const void*, const void*, uint32_t, uint32_t, size_t) { return false; } bool cactus_metal_encode_rope_full(void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, int) { return false; } bool cactus_metal_encode_maxpool1d(void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } From 3be16563fa91f31ef004e15d791dc594c4307bf3 Mon Sep 17 00:00:00 2001 From: jakmro Date: Mon, 6 Jul 2026 15:49:18 +0200 Subject: [PATCH 33/36] cleanup --- cactus-graph/src/execute.cpp | 1 - cactus-graph/src/metal_plan.cpp | 10 ---------- 2 files changed, 11 deletions(-) diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index e7081c5c2..9a282a5a4 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -1207,7 +1207,6 @@ MetalFusePlan* cactus_metal_plan_build(const std::vector& retype, const std::unordered_set* banned = nullptr); void cactus_metal_plan_free(MetalFusePlan* p); -void cactus_metal_plan_disable(MetalFusePlan* p); bool cactus_metal_plan_fold(MetalFusePlan* p, const std::vector>& nodes); const std::vector* cactus_metal_plan_exec_list(const MetalFusePlan* p); void* cactus_metal_plan_arena_ptr(const MetalFusePlan* p, size_t i); diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp index bf467f0e2..5a80117a1 100644 --- a/cactus-graph/src/metal_plan.cpp +++ b/cactus-graph/src/metal_plan.cpp @@ -1473,16 +1473,6 @@ void cactus_metal_plan_extend_last_use(const MetalFusePlan* p, std::vectoraction.begin(), p->action.end(), -1); - p->clusters.clear(); - p->fold_h = -1; - p->exec_list.clear(); - for (size_t i = 0; i < p->action.size(); ++i) p->exec_list.push_back((uint32_t)i); -} - int32_t cactus_metal_plan_action(const MetalFusePlan* p, size_t i) { return p && i < p->action.size() ? p->action[i] : -1; } From 39835a965a2d4466a71cc74744304d33301310e3 Mon Sep 17 00:00:00 2001 From: jakmro Date: Mon, 6 Jul 2026 18:25:30 +0200 Subject: [PATCH 34/36] more kernels --- cactus-engine/src/model.cpp | 3 +- cactus-graph/src/execute.cpp | 47 +++++++++ cactus-kernels/metal_backend.h | 8 ++ cactus-kernels/src/cactus_kernels.metal | 121 ++++++++++++++++++++++ cactus-kernels/src/metal_backend.mm | 51 ++++++++- cactus-kernels/src/metal_backend_stub.cpp | 4 + 6 files changed, 232 insertions(+), 2 deletions(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index c8a5cb6c9..25bfc938a 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -1353,7 +1353,8 @@ Model::ChunkedPrefillResult Model::run_chunked_prefill(const std::vectorgraph->get_node_op_type(id) == OpType::RECURRENT_CACHE_STATE; }); - if (has_recurrent_state && whole_chunks_end > effective_chunk) { + static const bool recurrent_multichunk = std::getenv("CACTUS_RECURRENT_MULTICHUNK") != nullptr; + if (has_recurrent_state && !recurrent_multichunk && whole_chunks_end > effective_chunk) { whole_chunks_end = effective_chunk; } const bool has_sliding_window_cache = any_cache_node([&](size_t id) { diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index 9a282a5a4..eb5a1ccba 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -869,6 +869,53 @@ static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const n if (cols == 0) return false; return cactus_metal_encode_softmax_rows(out.get_data(), in.get_data(), out.total_size / cols, cols); } + case OpType::GATED_DELTANET_DECODE: { + if (node.input_ids.size() != 6) return false; + const auto& q = get_input(node, 0, nodes, map); + const auto& k = get_input(node, 1, nodes, map); + const auto& v = get_input(node, 2, nodes, map); + const auto& g = get_input(node, 3, nodes, map); + const auto& b = get_input(node, 4, nodes, map); + const auto& st = get_input(node, 5, nodes, map); + if (!fp16(q) || !fp16(k) || !fp16(v) || !fp16(g) || !fp16(b) || !fp16(st) || !fp16(out)) return false; + if (q.shape.size() != 4 || v.shape.size() != 4 || st.shape.size() != 4) return false; + if (q.shape[1] != 1) return false; + size_t B = q.shape[0], Hq = q.shape[2], K = q.shape[3]; + size_t Hv = v.shape[2], V = v.shape[3]; + if (Hq == 0 || (Hv % Hq) != 0) return false; + if (st.shape[0] != B || st.shape[1] != K || st.shape[2] != Hv || st.shape[3] != V) return false; + if (out.total_size != B * (1 + K) * Hv * V) return false; + return cactus_metal_encode_deltanet_decode(out.get_data(), q.get_data(), k.get_data(), + v.get_data(), g.get_data(), b.get_data(), st.get_data(), + (uint32_t)B, (uint32_t)Hq, (uint32_t)Hv, (uint32_t)K, (uint32_t)V, node.params.scale); + } + case OpType::GATED_DELTANET_PREFILL: { + if (node.input_ids.size() != 6) return false; + const auto& q = get_input(node, 0, nodes, map); + const auto& k = get_input(node, 1, nodes, map); + const auto& v = get_input(node, 2, nodes, map); + const auto& g = get_input(node, 3, nodes, map); + const auto& b = get_input(node, 4, nodes, map); + const auto& st = get_input(node, 5, nodes, map); + if (!fp16(q) || !fp16(k) || !fp16(v) || !fp16(g) || !fp16(b) || !fp16(st) || !fp16(out)) return false; + if (q.shape.size() != 4 || v.shape.size() != 4 || st.shape.size() != 4) return false; + size_t B = q.shape[0], T = q.shape[1], Hq = q.shape[2], K = q.shape[3]; + size_t Hv = v.shape[2], V = v.shape[3]; + if (Hq == 0 || (Hv % Hq) != 0) return false; + if (st.shape[0] != B || st.shape[1] != K || st.shape[2] != Hv || st.shape[3] != V) return false; + if (out.total_size != B * (T + K) * Hv * V) return false; + return cactus_metal_encode_deltanet_prefill(out.get_data(), q.get_data(), k.get_data(), + v.get_data(), g.get_data(), b.get_data(), st.get_data(), + (uint32_t)B, (uint32_t)T, (uint32_t)Hq, (uint32_t)Hv, (uint32_t)K, (uint32_t)V, node.params.scale); + } + case OpType::RECURRENT_CACHE_WRITE: { + if (node.input_ids.size() < 2) return false; + const auto& src = get_input(node, 0, nodes, map); + BufferDesc& cache = nodes[map.at(node.input_ids[1])]->output_buffer; + if (!src.get_data() || !cache.get_data() || src.byte_size == 0) return false; + if (src.byte_size > cache.byte_size) return false; + return cactus_metal_encode_copy(cache.get_data(), src.get_data(), src.byte_size); + } case OpType::TOPK: { const auto& in = get_input(node, 0, nodes, map); if (!fp16(in) || out.precision != Precision::FP32) return false; diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h index 4bbe96f22..9505aad48 100644 --- a/cactus-kernels/metal_backend.h +++ b/cactus-kernels/metal_backend.h @@ -71,6 +71,14 @@ bool cactus_metal_encode_rope_pair(void* out, const void* x, const void* c, cons bool cactus_metal_encode_rope_pair_rms(void* out, const void* x, const void* w, const void* c, const void* s, uint32_t H, uint32_t D, float eps); +bool cactus_metal_encode_deltanet_decode(void* out, const void* q, const void* k, const void* v, + const void* g, const void* b, const void* s, + uint32_t B, uint32_t Hq, uint32_t Hv, + uint32_t K, uint32_t V, float scale); +bool cactus_metal_encode_deltanet_prefill(void* out, const void* q, const void* k, const void* v, + const void* g, const void* b, const void* s, + uint32_t B, uint32_t T, uint32_t Hq, uint32_t Hv, + uint32_t K, uint32_t V, float scale); bool cactus_metal_encode_rms2_add_clip(void* out, const void* a, const void* wa, const void* b, const void* wb, size_t dim, float eps_a, float eps_b); diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 5079d15a7..14ceac9e9 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -3669,3 +3669,124 @@ kernel void rms2_add_clip_f16( y[i] = (half)clamp((float)ya + (float)yb, -65500.0f, 65500.0f); } } + +kernel void gated_deltanet_decode_f16( + device const half* q [[buffer(0)]], + device const half* k [[buffer(1)]], + device const half* v [[buffer(2)]], + device const half* g [[buffer(3)]], + device const half* b [[buffer(4)]], + device const half* s [[buffer(5)]], + device half* y [[buffer(6)]], + constant uint& Hq [[buffer(7)]], + constant uint& Hv [[buffer(8)]], + constant uint& K [[buffer(9)]], + constant uint& V [[buffer(10)]], + constant float& scale [[buffer(11)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint t [[thread_index_in_threadgroup]], + threadgroup half* kq [[threadgroup(0)]]) +{ + const uint h = tgp.x, batch = tgp.y; + const uint qk_head = h / (Hv / Hq); + const uint qk_base = (batch * Hq + qk_head) * K; + threadgroup half* kk = kq; + threadgroup half* qq = kq + K; + for (uint i = t; i < K; i += V) { + kk[i] = k[qk_base + i]; + qq[i] = q[qk_base + i]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (t >= V) return; + + float gate_log = (float)g[batch * Hv + h]; + if (!isfinite(gate_log)) gate_log = -20.0f; + float beta = (float)b[batch * Hv + h]; + beta = isfinite(beta) ? clamp(beta, 0.0f, 1.0f) : 0.0f; + const float alpha = exp(clamp(gate_log, -20.0f, 6.0f)); + + const size_t s_stride = (size_t)Hv * V; + device const half* sp = s + ((size_t)batch * K) * s_stride + (size_t)h * V + t; + float proj = 0.0f; + for (uint kd = 0; kd < K; ++kd) proj += (float)sp[kd * s_stride] * (float)kk[kd]; + const float delta = ((float)v[((size_t)batch * Hv + h) * V + t] - alpha * proj) * beta; + + device half* yp = y + ((size_t)batch * (1 + K) + 1) * s_stride + (size_t)h * V + t; + float acc = 0.0f; + for (uint kd = 0; kd < K; ++kd) { + float s_new = (float)sp[kd * s_stride] * alpha + (float)kk[kd] * delta; + if (!isfinite(s_new)) s_new = 0.0f; + yp[kd * s_stride] = (half)s_new; + acc += s_new * (float)qq[kd]; + } + if (!isfinite(acc)) acc = 0.0f; + y[((size_t)batch * (1 + K)) * s_stride + (size_t)h * V + t] = (half)(acc * scale); +} + +kernel void gated_deltanet_prefill_f16( + device const half* q [[buffer(0)]], + device const half* k [[buffer(1)]], + device const half* v [[buffer(2)]], + device const half* g [[buffer(3)]], + device const half* b [[buffer(4)]], + device const half* s [[buffer(5)]], + device half* y [[buffer(6)]], + device float* st [[buffer(7)]], + constant uint& T [[buffer(8)]], + constant uint& Hq [[buffer(9)]], + constant uint& Hv [[buffer(10)]], + constant uint& K [[buffer(11)]], + constant uint& V [[buffer(12)]], + constant float& scale [[buffer(13)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint t [[thread_index_in_threadgroup]], + threadgroup half* kq [[threadgroup(0)]]) +{ + const uint h = tgp.x, batch = tgp.y; + const uint qk_head = h / (Hv / Hq); + threadgroup half* kk = kq; + threadgroup half* qq = kq + K; + const size_t hv_stride = (size_t)Hv * V; + device float* col = st + (((size_t)batch * Hv + h) * K) * V + t; + + if (t < V) { + device const half* sp = s + ((size_t)batch * K) * hv_stride + (size_t)h * V + t; + for (uint kd = 0; kd < K; ++kd) col[(size_t)kd * V] = (float)sp[kd * hv_stride]; + } + + for (uint step = 0; step < T; ++step) { + threadgroup_barrier(mem_flags::mem_threadgroup); + const size_t qk_base = ((size_t)(batch * T + step) * Hq + qk_head) * K; + for (uint i = t; i < K; i += V) { + kk[i] = k[qk_base + i]; + qq[i] = q[qk_base + i]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (t >= V) continue; + + float gate_log = (float)g[(size_t)(batch * T + step) * Hv + h]; + if (!isfinite(gate_log)) gate_log = -20.0f; + float beta = (float)b[(size_t)(batch * T + step) * Hv + h]; + beta = isfinite(beta) ? clamp(beta, 0.0f, 1.0f) : 0.0f; + const float alpha = exp(clamp(gate_log, -20.0f, 6.0f)); + + float proj = 0.0f; + for (uint kd = 0; kd < K; ++kd) proj += col[(size_t)kd * V] * (float)kk[kd]; + const float delta = ((float)v[((size_t)(batch * T + step) * Hv + h) * V + t] - alpha * proj) * beta; + + float acc = 0.0f; + for (uint kd = 0; kd < K; ++kd) { + float s_new = col[(size_t)kd * V] * alpha + (float)kk[kd] * delta; + if (!isfinite(s_new)) s_new = 0.0f; + col[(size_t)kd * V] = s_new; + acc += s_new * (float)qq[kd]; + } + if (!isfinite(acc)) acc = 0.0f; + y[((size_t)(batch * (T + K) + step)) * hv_stride + (size_t)h * V + t] = (half)(acc * scale); + } + + if (t < V) { + for (uint kd = 0; kd < K; ++kd) + y[((size_t)(batch * (T + K) + T + kd)) * hv_stride + (size_t)h * V + t] = (half)col[(size_t)kd * V]; + } +} diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index dc314cdac..d6b79d576 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -66,7 +66,7 @@ id psoTopkRows=nil, psoMoeT=nil, psoMoeUp=nil; id psoMoeT2=nil, psoMoeDownAcc=nil, psoRopePair=nil; id psoRmsScale=nil, psoSoftmaxTopk=nil, psoRopePairRms=nil; - id psoRms2AddClip=nil; + id psoRms2AddClip=nil, psoDeltanet=nil, psoDeltanetPre=nil; id dummy=nil; bool ok = false; @@ -132,6 +132,7 @@ psoMoeDownAcc=pso("cq4_moe_gemv_down_acc"); psoRopePair=pso("rope_pair_f16"); psoRmsScale=pso("rms_norm_scale_f16"); psoSoftmaxTopk=pso("softmax_topk_f16"); psoRopePairRms=pso("rope_pair_rms_f16"); psoRms2AddClip=pso("rms2_add_clip_f16"); + psoDeltanet=pso("gated_deltanet_decode_f16"); psoDeltanetPre=pso("gated_deltanet_prefill_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; ok = psoT&&psoG&&psoTm&&psoGm&&psoRotW&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoArgmax&&psoGather; @@ -1150,6 +1151,54 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void return true; } +bool cactus_metal_encode_deltanet_decode(void* out, const void* q, const void* k, const void* v, + const void* g, const void* b, const void* s, + uint32_t B, uint32_t Hq, uint32_t Hv, + uint32_t K, uint32_t V, float scale) { + if (!ctx().ok || !ctx().psoDeltanet) return false; + if (B == 0 || Hq == 0 || Hv == 0 || K == 0 || V == 0 || V > 1024 || (Hv % Hq) != 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoDeltanet]; + setBufAt(q, (size_t)B*Hq*K*2, 0); + setBufAt(k, (size_t)B*Hq*K*2, 1); + setBufAt(v, (size_t)B*Hv*V*2, 2); + setBufAt(g, (size_t)B*Hv*2, 3); + setBufAt(b, (size_t)B*Hv*2, 4); + setBufAt(s, (size_t)B*K*Hv*V*2, 5); + setBufAt(out, (size_t)B*(1+K)*Hv*V*2, 6); + [g_enc setBytes:&Hq length:4 atIndex:7]; [g_enc setBytes:&Hv length:4 atIndex:8]; + [g_enc setBytes:&K length:4 atIndex:9]; [g_enc setBytes:&V length:4 atIndex:10]; + [g_enc setBytes:&scale length:4 atIndex:11]; + [g_enc setThreadgroupMemoryLength:(size_t)K*4 atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(Hv, B, 1) threadsPerThreadgroup:MTLSizeMake(V, 1, 1)]; + return true; +} + +bool cactus_metal_encode_deltanet_prefill(void* out, const void* q, const void* k, const void* v, + const void* g, const void* b, const void* s, + uint32_t B, uint32_t T, uint32_t Hq, uint32_t Hv, + uint32_t K, uint32_t V, float scale) { + if (!ctx().ok || !ctx().psoDeltanetPre) return false; + if (B == 0 || T == 0 || Hq == 0 || Hv == 0 || K == 0 || V == 0 || V > 1024 || (Hv % Hq) != 0) return false; + ensureEncoder(); + id scratch = recycled((size_t)B*Hv*K*V*sizeof(float)); + [g_enc setComputePipelineState:ctx().psoDeltanetPre]; + setBufAt(q, (size_t)B*T*Hq*K*2, 0); + setBufAt(k, (size_t)B*T*Hq*K*2, 1); + setBufAt(v, (size_t)B*T*Hv*V*2, 2); + setBufAt(g, (size_t)B*T*Hv*2, 3); + setBufAt(b, (size_t)B*T*Hv*2, 4); + setBufAt(s, (size_t)B*K*Hv*V*2, 5); + setBufAt(out, (size_t)B*(T+K)*Hv*V*2, 6); + [g_enc setBuffer:scratch offset:0 atIndex:7]; + [g_enc setBytes:&T length:4 atIndex:8]; [g_enc setBytes:&Hq length:4 atIndex:9]; + [g_enc setBytes:&Hv length:4 atIndex:10]; [g_enc setBytes:&K length:4 atIndex:11]; + [g_enc setBytes:&V length:4 atIndex:12]; [g_enc setBytes:&scale length:4 atIndex:13]; + [g_enc setThreadgroupMemoryLength:(size_t)K*4 atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(Hv, B, 1) threadsPerThreadgroup:MTLSizeMake(V, 1, 1)]; + return true; +} + bool cactus_metal_encode_rms2_add_clip(void* out, const void* a, const void* wa, const void* b, const void* wb, size_t dim, float eps_a, float eps_b) { diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp index 766b212ec..9ecafc05b 100644 --- a/cactus-kernels/src/metal_backend_stub.cpp +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -117,6 +117,10 @@ bool cactus_metal_encode_rope_pair(void*, const void*, const void*, const void*, uint32_t, uint32_t) { return false; } bool cactus_metal_encode_rope_pair_rms(void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_deltanet_decode(void*, const void*, const void*, const void*, + const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_deltanet_prefill(void*, const void*, const void*, const void*, + const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float) { return false; } bool cactus_metal_encode_rms2_add_clip(void*, const void*, const void*, const void*, const void*, size_t, float, float) { return false; } bool cactus_metal_encode_rms_norm_scale(void*, const void*, const void*, From 7cc26603f1f1993d9191ae915a8a35dec3391059 Mon Sep 17 00:00:00 2001 From: jakmro Date: Tue, 7 Jul 2026 14:55:16 +0200 Subject: [PATCH 35/36] ram optimizations --- cactus-engine/src/model.cpp | 3 +- cactus-kernels/src/cactus_kernels.metal | 371 +++++++------- cactus-kernels/src/metal_backend.mm | 634 +++++++++++------------- 3 files changed, 472 insertions(+), 536 deletions(-) diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 25bfc938a..c8a5cb6c9 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -1353,8 +1353,7 @@ Model::ChunkedPrefillResult Model::run_chunked_prefill(const std::vectorgraph->get_node_op_type(id) == OpType::RECURRENT_CACHE_STATE; }); - static const bool recurrent_multichunk = std::getenv("CACTUS_RECURRENT_MULTICHUNK") != nullptr; - if (has_recurrent_state && !recurrent_multichunk && whole_chunks_end > effective_chunk) { + if (has_recurrent_state && whole_chunks_end > effective_chunk) { whole_chunks_end = effective_chunk; } const bool has_sliding_window_cache = any_cache_node([&](size_t id) { diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal index 14ceac9e9..01c3b8b4d 100644 --- a/cactus-kernels/src/cactus_kernels.metal +++ b/cactus-kernels/src/cactus_kernels.metal @@ -39,6 +39,52 @@ static inline float cq4_dot64(half4 c0, half4 c1, half4 c2, half4 c3, + (float)c3.x*cb[w[3]&0xF] + (float)c3.y*cb[(w[3]>>4)&0xF] + (float)c3.z*cb[(w[3]>>8)&0xF] + (float)c3.w*cb[(w[3]>>12)&0xF]; } +static inline device const uchar4* cq4_il_base(device const uchar* packed, uint n, uint ng, uint g, uint pgb) { + return (device const uchar4*)(packed + ((size_t)(n>>2)*ng+g)*4u*(size_t)pgb + (n&3u)*4u); +} + +static inline float cq4_dot64_il(half4 c0, half4 c1, half4 c2, half4 c3, + uchar4 A, uchar4 B, threadgroup const float* cb) { + return (float)c0.x*cb[A.x&0xFu] + (float)c0.y*cb[A.y&0xFu] + (float)c0.z*cb[A.z&0xFu] + (float)c0.w*cb[A.w&0xFu] + + (float)c1.x*cb[A.x>>4] + (float)c1.y*cb[A.y>>4] + (float)c1.z*cb[A.z>>4] + (float)c1.w*cb[A.w>>4] + + (float)c2.x*cb[B.x&0xFu] + (float)c2.y*cb[B.y&0xFu] + (float)c2.z*cb[B.z&0xFu] + (float)c2.w*cb[B.w&0xFu] + + (float)c3.x*cb[B.x>>4] + (float)c3.y*cb[B.y>>4] + (float)c3.z*cb[B.z>>4] + (float)c3.w*cb[B.w>>4]; +} + +static inline float cq4_norm_at(device const half* norms, uint n, uint ng, uint g, uint il) { + return (float)(il != 0u ? norms[(((size_t)(n>>2)*ng+g)<<2) + (n&3u)] + : norms[(size_t)n*ng+g]); +} + +static inline uint cq4_il_nib(device const uchar* packed, uint n, uint ng, uint g, uint pgb, uint e) { + uint v = e>>4, j = e&15u, sub = j>>2, b = j&3u; + device const uchar* panel = packed + ((size_t)(n>>2)*ng+g)*4u*(size_t)pgb; + uchar by = panel[(2u*v + (sub>>1))*16u + (n&3u)*4u + b]; + return (sub&1u) ? (uint)(by>>4) : (uint)(by&0xFu); +} + +static inline uint cq_il_idx(device const uchar* packed, uint n, uint ng, uint g, uint pgb, uint e, uint bits) { + if (bits == 4u) return cq4_il_nib(packed, n, ng, g, pgb, e); + device const uchar* panel = packed + ((size_t)(n>>2)*ng+g)*4u*(size_t)pgb; + uint r = n&3u; + if (bits == 2u) { + uchar by = panel[(e>>4)*16u + ((e&15u)>>2)*4u + r]; + return ((uint)by >> (2u*(e&3u))) & 3u; + } + device const uchar* ch = panel + (e>>2)*6u; + ulong word = (ulong)ch[0] | ((ulong)ch[1]<<8) | ((ulong)ch[2]<<16) + | ((ulong)ch[3]<<24) | ((ulong)ch[4]<<32) | ((ulong)ch[5]<<40); + return (uint)((word >> (r*12u + (e&3u)*3u)) & 7u); +} + +static inline float cq4_dot64_ilw(half4 c0, half4 c1, half4 c2, half4 c3, + uint A, uint B, threadgroup const float* cb) { + return (float)c0.x*cb[A&0xFu] + (float)c0.y*cb[(A>>8)&0xFu] + (float)c0.z*cb[(A>>16)&0xFu] + (float)c0.w*cb[(A>>24)&0xFu] + + (float)c1.x*cb[(A>>4)&0xFu] + (float)c1.y*cb[(A>>12)&0xFu] + (float)c1.z*cb[(A>>20)&0xFu] + (float)c1.w*cb[A>>28] + + (float)c2.x*cb[B&0xFu] + (float)c2.y*cb[(B>>8)&0xFu] + (float)c2.z*cb[(B>>16)&0xFu] + (float)c2.w*cb[(B>>24)&0xFu] + + (float)c3.x*cb[(B>>4)&0xFu] + (float)c3.y*cb[(B>>12)&0xFu] + (float)c3.z*cb[(B>>20)&0xFu] + (float)c3.w*cb[B>>28]; +} + kernel void cq4_transform( device const half* x [[buffer(0)]], device const half* recip [[buffer(1)]], @@ -131,6 +177,7 @@ kernel void cq4_gemv( constant uint& num_groups [[buffer(6)]], constant uint& pgb [[buffer(7)]], constant uint& N [[buffer(8)]], + constant uint& il [[buffer(9)]], uint tg [[threadgroup_position_in_grid]], uint sgid [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], @@ -147,17 +194,24 @@ kernel void cq4_gemv( uint g = base/gs; uint off = base - g*gs; device const half4* cbase=(device const half4*)(code + g*gs + off); - device const ushort4* pr=(device const ushort4*)(packed + ((size_t)n*num_groups+g)*pgb + off/2u); - ushort4 w=pr[0]; - float p = 0; - #pragma clang loop unroll(full) - for (uint q=0;q<4;++q){ - half4 c=cbase[q]; - ushort ww = w[q]; - p += (float)c.x*cb[ww&0xF] + (float)c.y*cb[(ww>>4)&0xF] - + (float)c.z*cb[(ww>>8)&0xF] + (float)c.w*cb[(ww>>12)&0xF]; + if (il != 0u) { + device const uchar4* pp = cq4_il_base(packed, n, num_groups, g, pgb); + uchar4 A = pp[(off>>4)*8u], B = pp[(off>>4)*8u+4u]; + float p = cq4_dot64_il(cbase[0], cbase[1], cbase[2], cbase[3], A, B, cb); + acc += (float)norms[(((size_t)(n>>2)*num_groups+g)<<2) + (n&3u)]*p; + } else { + device const ushort4* pr=(device const ushort4*)(packed + ((size_t)n*num_groups+g)*pgb + off/2u); + ushort4 w=pr[0]; + float p = 0; + #pragma clang loop unroll(full) + for (uint q=0;q<4;++q){ + half4 c=cbase[q]; + ushort ww = w[q]; + p += (float)c.x*cb[ww&0xF] + (float)c.y*cb[(ww>>4)&0xF] + + (float)c.z*cb[(ww>>8)&0xF] + (float)c.w*cb[(ww>>12)&0xF]; + } + acc += (float)norms[(size_t)n*num_groups+g]*p; } - acc += (float)norms[(size_t)n*num_groups+g]*p; } acc=simd_sum(acc); if (lane==0) y[n]=(half)acc; @@ -174,6 +228,7 @@ kernel void cq4_gemv_mr( constant uint& num_groups [[buffer(6)]], constant uint& pgb [[buffer(7)]], constant uint& N [[buffer(8)]], + constant uint& il [[buffer(9)]], uint tg [[threadgroup_position_in_grid]], uint sgid [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], @@ -192,13 +247,25 @@ kernel void cq4_gemv_mr( uint g = base/gs, off = base - g*gs; device const half4* cbase = (device const half4*)(code + g*gs + off); half4 c0=cbase[0], c1=cbase[1], c2=cbase[2], c3=cbase[3]; - #pragma clang loop unroll(full) - for (uint r=0;r>2)*num_groups+g)*4u*(size_t)pgb + + (n0&3u)*4u + (off>>4)*32u; + uint2 Aw = ((device const uint2*)pan)[0]; + uint2 Bw = ((device const uint2*)(pan+16u))[0]; + #pragma clang loop unroll(full) + for (uint r=0;r>2)*num_groups+g)<<2) + (n0&3u) + r]*p; + } + } else { + #pragma clang loop unroll(full) + for (uint r=0;r>2)*ng+g)*256u + (n0&3u)*4u + (off>>4)*32u; + uint2 Aw = ((device const uint2*)pan)[0]; + uint2 Bw = ((device const uint2*)(pan+16u))[0]; #pragma clang loop unroll(full) for (uint r=0u;r<2u;r++){ - uint n = n0+r; - device const ushort4* pr = (device const ushort4*)(packed + ((size_t)n*ng+g)*64u + off/2u); - ushort4 w = pr[0]; - float p = cq4_dot64(c0, c1, c2, c3, w, cb); - acc[r] += (float)norms[(size_t)n*ng+g]*p; + float p = cq4_dot64_ilw(c0, c1, c2, c3, r?Aw.y:Aw.x, r?Bw.y:Bw.x, cb); + acc[r] += (float)norms[(((size_t)(n0>>2)*ng+g)<<2) + (n0&3u) + r]*p; } } #pragma clang loop unroll(full) @@ -352,13 +419,13 @@ kernel void cq4_gemv_cat( uint g = base/128u, off = base - g*128u; device const half4* cbase = (device const half4*)(code + g*128u + off); half4 c0=cbase[0], c1=cbase[1], c2=cbase[2], c3=cbase[3]; + device const uchar* pan = packed + ((size_t)(nl0>>2)*ng+g)*256u + (nl0&3u)*4u + (off>>4)*32u; + uint2 Aw = ((device const uint2*)pan)[0]; + uint2 Bw = ((device const uint2*)(pan+16u))[0]; #pragma clang loop unroll(full) for (uint r=0;r<2;r++){ - uint n = nl0+r; - device const ushort4* pr = (device const ushort4*)(packed + ((size_t)n*ng+g)*64u + off/2u); - ushort4 w = pr[0]; - float p = cq4_dot64(c0, c1, c2, c3, w, cb); - acc[r] += (float)norms[(size_t)n*ng+g]*p; + float p = cq4_dot64_ilw(c0, c1, c2, c3, r?Aw.y:Aw.x, r?Bw.y:Bw.x, cb); + acc[r] += (float)norms[(((size_t)(nl0>>2)*ng+g)<<2) + (nl0&3u) + r]*p; } } #pragma clang loop unroll(full) @@ -379,6 +446,7 @@ kernel void cq_gemv_mr_lowbit( constant uint& pgb [[buffer(7)]], constant uint& N [[buffer(8)]], constant uint& bits [[buffer(9)]], + constant uint& il [[buffer(10)]], uint tg [[threadgroup_position_in_grid]], uint sgid [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], @@ -402,17 +470,37 @@ kernel void cq_gemv_mr_lowbit( #pragma clang loop unroll(full) for (uint r=0;r>2)*num_groups+g)*4u*(size_t)pgb + byte_off*4u; + uint rr = n&3u; + w = (uint)pan[rr] | ((uint)pan[4u+rr]<<8) | ((uint)pan[8u+rr]<<16) | ((uint)pan[12u+rr]<<24); + } else { + w = *(device const uint*)(packed + ((size_t)n*num_groups+g)*pgb + byte_off); + } w0=cb[w&3u]; w1=cb[(w>>2)&3u]; w2=cb[(w>>4)&3u]; w3=cb[(w>>6)&3u]; w4=cb[(w>>8)&3u]; w5=cb[(w>>10)&3u]; w6=cb[(w>>12)&3u]; w7=cb[(w>>14)&3u]; w8=cb[(w>>16)&3u]; w9=cb[(w>>18)&3u]; wa=cb[(w>>20)&3u]; wb=cb[(w>>22)&3u]; wc=cb[(w>>24)&3u]; wd=cb[(w>>26)&3u]; we=cb[(w>>28)&3u]; wf=cb[(w>>30)&3u]; } else { - device const ushort* ps = (device const ushort*)pr; - ulong v = (ulong)ps[0] | ((ulong)ps[1] << 16) | ((ulong)ps[2] << 32); + ulong v; + if (il != 0u) { + device const uchar* pan = packed + ((size_t)(n>>2)*num_groups+g)*4u*(size_t)pgb + byte_off*4u; + uint rr = n&3u; + v = 0; + #pragma clang loop unroll(full) + for (uint q=0;q<4u;q++){ + device const uchar* ch = pan + q*6u; + ulong word = (ulong)ch[0] | ((ulong)ch[1]<<8) | ((ulong)ch[2]<<16) + | ((ulong)ch[3]<<24) | ((ulong)ch[4]<<32) | ((ulong)ch[5]<<40); + v |= ((word >> (rr*12u)) & 0xFFFul) << (12u*q); + } + } else { + device const ushort* ps = (device const ushort*)(packed + ((size_t)n*num_groups+g)*pgb + byte_off); + v = (ulong)ps[0] | ((ulong)ps[1] << 16) | ((ulong)ps[2] << 32); + } w0=cb[v&7u]; w1=cb[(v>>3)&7u]; w2=cb[(v>>6)&7u]; w3=cb[(v>>9)&7u]; w4=cb[(v>>12)&7u]; w5=cb[(v>>15)&7u]; w6=cb[(v>>18)&7u]; w7=cb[(v>>21)&7u]; w8=cb[(v>>24)&7u]; w9=cb[(v>>27)&7u]; wa=cb[(v>>30)&7u]; wb=cb[(v>>33)&7u]; @@ -422,7 +510,7 @@ kernel void cq_gemv_mr_lowbit( + (float)c1.x*w4 + (float)c1.y*w5 + (float)c1.z*w6 + (float)c1.w*w7 + (float)c2.x*w8 + (float)c2.y*w9 + (float)c2.z*wa + (float)c2.w*wb + (float)c3.x*wc + (float)c3.y*wd + (float)c3.z*we + (float)c3.w*wf; - acc[r] += (float)norms[(size_t)n*num_groups+g]*pacc; + acc[r] += cq4_norm_at(norms, n, num_groups, g, il)*pacc; } } #pragma clang loop unroll(full) @@ -432,28 +520,6 @@ kernel void cq_gemv_mr_lowbit( } } -kernel void cq_dequant_lowbit_f16( - device const uchar* packed [[buffer(0)]], - device const half* codebook [[buffer(1)]], - device const half* norms [[buffer(2)]], - device half* wf16 [[buffer(3)]], - constant uint& gs [[buffer(4)]], - constant uint& num_groups [[buffer(5)]], - constant uint& pgb [[buffer(6)]], - constant uint& N [[buffer(7)]], - constant uint& bits [[buffer(8)]], - uint gid [[thread_position_in_grid]]) -{ - if (gid >= N*num_groups) return; - uint n = gid / num_groups, g = gid % num_groups; - uint K = num_groups*gs; - float nrm = (float)norms[(size_t)n*num_groups + g]; - device const uchar* pr = packed + ((size_t)n*num_groups + g)*pgb; - device half* wr = wf16 + (size_t)n*K + (size_t)g*gs; - for (uint e=0; e= N) return; float acc = 0.f; for (uint g = 0u; g < num_groups; ++g) { - uchar b = packed[((size_t)n*num_groups + g)*32u + lane]; + uchar b = packed[((size_t)(n>>2)*num_groups + g)*128u + (lane>>2)*16u + (lane&3u)*4u + (n&3u)]; device const char* a = act_i8 + (size_t)g*128u + lane*4u; int partial = (int)a[0]*cb[b & 3u] + (int)a[1]*cb[(b >> 2) & 3u] + (int)a[2]*cb[(b >> 4) & 3u] + (int)a[3]*cb[(b >> 6) & 3u]; int idot = simd_sum(partial); if (lane == 0u) { - float ns = (float)norms[(size_t)n*num_groups + g] * cb_scale; + float ns = (float)norms[(((size_t)(n>>2)*num_groups+g)<<2) + (n&3u)] * cb_scale; float sc = ns * act_scales[g]; acc = fma((float)idot, sc, acc); } @@ -539,56 +605,6 @@ kernel void cq4_transform_m( for (uint k=t; k=N) return; - uint m0 = tg.y*CQ4_MT; - uint K = num_groups*gs; - float acc[CQ4_MT]; - for (uint mi=0; mi>4)&0xF]; - wv[q*4+2]=cb[(ww>>8)&0xF]; wv[q*4+3]=cb[(ww>>12)&0xF]; } - for (uint mi=0; mi>2)*num_groups+g)<<2) + (nrow&3u)]:0.f; + float wv[16]; + if (nrow>4; + uchar4 A = pp[v4*8u], B = pp[v4*8u+4u]; + wv[0]=cb[A.x&0xFu]; wv[1]=cb[A.y&0xFu]; wv[2]=cb[A.z&0xFu]; wv[3]=cb[A.w&0xFu]; + wv[4]=cb[A.x>>4]; wv[5]=cb[A.y>>4]; wv[6]=cb[A.z>>4]; wv[7]=cb[A.w>>4]; + wv[8]=cb[B.x&0xFu]; wv[9]=cb[B.y&0xFu]; wv[10]=cb[B.z&0xFu]; wv[11]=cb[B.w&0xFu]; + wv[12]=cb[B.x>>4]; wv[13]=cb[B.y>>4]; wv[14]=cb[B.z>>4]; wv[15]=cb[B.w>>4]; + } for (uint i=0;i<16u;i++){ uint sx=2u*il0+i/8u, sy=(tl/NL0)/8u, lx=(tl/NL0)%8u, ly=i%8u, ib=8u*sx+sy; - float v=0.f; uint e=e0+16u*il0+i; - if (loop_k+16u*il0+i>1)]; - uint nib=(e&1u)?(uint)(by>>4):(uint)(by&0xF); v=nrm*cb[nib]; } + float v=0.f; + if (loop_k+16u*il0+i= N*num_groups) return; - uint n = gid / num_groups, g = gid % num_groups; - uint K = num_groups*gs; - float nrm = (float)norms[(size_t)n*num_groups + g]; - device const uchar* pr = packed + ((size_t)n*num_groups + g)*pgb; - device half* wr = wf16 + (size_t)n*K + (size_t)g*gs; - for (uint e=0; e>1]; - uint nib = (e&1u)?(uint)(by>>4):(uint)(by&0xF); - wr[e] = (half)(nrm * (float)codebook[nib]); - } -} - kernel void cq4_gemm_dense_f16( device const half* code [[buffer(0)]], device const half* wf16 [[buffer(1)]], @@ -766,6 +766,7 @@ kernel void emb_ortho_wide(device const uchar* packed [[buffer(0)]], constant uint& K [[buffer(6)]], constant uint& row [[buffer(7)]], constant float& oscale [[buffer(8)]], constant uint& bits [[buffer(9)]], + constant uint& il [[buffer(10)]], uint tg [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], uint sg [[simdgroup_index_in_threadgroup]], @@ -773,7 +774,8 @@ kernel void emb_ortho_wide(device const uchar* packed [[buffer(0)]], threadgroup float* dq [[threadgroup(0)]]) { uint pgb = (K*bits + 7u)/8u; device const uchar* prow = packed + (size_t)row*pgb; - for (uint i=t; i>1]; uint idx=(i&1u)?(uint)(b>>4):(uint)(b&0xF); dq[i]=(float)codebook[idx]; } - threadgroup_barrier(mem_flags::mem_threadgroup); - float nrm = (float)norms[row]; - for (uint j=t; j= N) return; float gv[4] = {0,0,0,0}, uv[4] = {0,0,0,0}; for (int m = 0; m < 2; ++m) { - device const uchar* pb = (m==0) ? pk1 + (size_t)e*N*ng*pgb : pk3 + (size_t)e*N*ng*pgb; - device const half* nb = (m==0) ? nm1 + (size_t)e*N*ng : nm3 + (size_t)e*N*ng; + device const uchar* pb = (m==0) ? pk1 + (size_t)e*estride1 : pk3 + (size_t)e*estride3; + device const half* nb = (m==0) ? (device const half*)((device const uchar*)nm1 + (size_t)e*estride1) + : (device const half*)((device const uchar*)nm3 + (size_t)e*estride3); device const half* cd = (m==0) ? code1 + (size_t)slotg*K : code3 + (size_t)slotg*K; threadgroup float* cb = (m==0) ? c1 : c3; float a0=0, a1=0, a2=0, a3=0; @@ -3370,14 +3364,16 @@ kernel void cq4_moe_gemv_up( uint gg = base/128u, off = base - gg*128u; device const half4* cp = (device const half4*)(cd + gg*128u + off); half4 c0=cp[0], c1v=cp[1], c2v=cp[2], c3v=cp[3]; - device const ushort4* q0 = (device const ushort4*)(pb + ((size_t)(n0+0u)*ng+gg)*pgb + off/2u); - device const ushort4* q1 = (device const ushort4*)(pb + ((size_t)(n0+1u)*ng+gg)*pgb + off/2u); - device const ushort4* q2 = (device const ushort4*)(pb + ((size_t)(n0+2u)*ng+gg)*pgb + off/2u); - device const ushort4* q3 = (device const ushort4*)(pb + ((size_t)(n0+3u)*ng+gg)*pgb + off/2u); - a0 += (float)nb[(size_t)(n0+0u)*ng+gg]*cq4_dot64(c0, c1v, c2v, c3v, q0[0], cb); - a1 += (float)nb[(size_t)(n0+1u)*ng+gg]*cq4_dot64(c0, c1v, c2v, c3v, q1[0], cb); - a2 += (float)nb[(size_t)(n0+2u)*ng+gg]*cq4_dot64(c0, c1v, c2v, c3v, q2[0], cb); - a3 += (float)nb[(size_t)(n0+3u)*ng+gg]*cq4_dot64(c0, c1v, c2v, c3v, q3[0], cb); + device const uchar* pan = pb + ((size_t)(n0>>2)*ng+gg)*256u + (off>>4)*32u; + uint2 A01 = ((device const uint2*)pan)[0]; + uint2 A23 = ((device const uint2*)(pan+8u))[0]; + uint2 B01 = ((device const uint2*)(pan+16u))[0]; + uint2 B23 = ((device const uint2*)(pan+24u))[0]; + size_t nb4 = ((size_t)(n0>>2)*ng+gg)<<2; + a0 += (float)nb[nb4+0u]*cq4_dot64_ilw(c0, c1v, c2v, c3v, A01.x, B01.x, cb); + a1 += (float)nb[nb4+1u]*cq4_dot64_ilw(c0, c1v, c2v, c3v, A01.y, B01.y, cb); + a2 += (float)nb[nb4+2u]*cq4_dot64_ilw(c0, c1v, c2v, c3v, A23.x, B23.x, cb); + a3 += (float)nb[nb4+3u]*cq4_dot64_ilw(c0, c1v, c2v, c3v, A23.y, B23.y, cb); } a0 = simd_sum(a0); a1 = simd_sum(a1); a2 = simd_sum(a2); a3 = simd_sum(a3); if (m == 0) { gv[0]=a0; gv[1]=a1; gv[2]=a2; gv[3]=a3; } @@ -3408,6 +3404,8 @@ kernel void cq4_moe_transform2( device half* code3 [[buffer(11)]], constant uint& K [[buffer(12)]], constant uint& topk_k [[buffer(13)]], + constant uint& estride1 [[buffer(14)]], + constant uint& estride3 [[buffer(15)]], uint3 tgp [[threadgroup_position_in_grid]], uint lane [[thread_index_in_simdgroup]], threadgroup float* zmem [[threadgroup(0)]]) @@ -3416,10 +3414,11 @@ kernel void cq4_moe_transform2( const bool w3 = tgp.y >= topk_k; const uint slot = w3 ? tgp.y - topk_k : tgp.y; const uint e = (uint)(topk[(size_t)tok*topk_k + slot] + 0.5f); - device const half* recip = (w3 ? recip_c3 : recip_c1) + (size_t)e*K; - device const char* lsign = (w3 ? lsign_c3 : lsign_c1) + (size_t)e*128u; - device const char* rsign = (w3 ? rsign_c3 : rsign_c1) + (size_t)e*128u; - device const uint* perm = (w3 ? perm_c3 : perm_c1 ) + (size_t)e*128u; + const uint es = w3 ? estride3 : estride1; + device const half* recip = (device const half*)((device const uchar*)(w3 ? recip_c3 : recip_c1) + (size_t)e*es); + device const char* lsign = (w3 ? lsign_c3 : lsign_c1) + (size_t)e*es; + device const char* rsign = (w3 ? rsign_c3 : rsign_c1) + (size_t)e*es; + device const uint* perm = (device const uint*)((device const uchar*)(w3 ? perm_c3 : perm_c1) + (size_t)e*es); device const half* xs = x + (size_t)tok*K; uint b = g*128u + lane*4u; uint k = lane*4u; @@ -3451,6 +3450,7 @@ kernel void cq4_moe_gemv_down_acc( constant float& eps [[buffer(11)]], constant float& scaling [[buffer(12)]], constant uint& E [[buffer(13)]], + constant uint& estride [[buffer(14)]], uint3 tgp [[threadgroup_position_in_grid]], uint sgid [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]], @@ -3472,7 +3472,7 @@ kernel void cq4_moe_gemv_down_acc( if (tl < topk_k*16u) { uint s = tl >> 4, ci = tl & 15u; uint e = (uint)(tk[s] + 0.5f); - cbs[tl] = (float)cb2[(size_t)e*16u + ci]; + cbs[tl] = (float)((device const half*)((device const uchar*)cb2 + (size_t)e*estride))[ci]; } threadgroup_barrier(mem_flags::mem_threadgroup); float denom = 1.0f; @@ -3481,7 +3481,7 @@ kernel void cq4_moe_gemv_down_acc( for (uint j = 0u; j < topk_k; ++j) s += (float)pr[(uint)(tk[j] + 0.5f)]; denom = s + eps; } - const uint ng = K/128u, pgb = 64u; + const uint ng = K/128u; uint n0 = (tgp.x*8u + sgid)*2u; if (n0 >= N) return; half res[2] = {0.0h, 0.0h}; @@ -3491,8 +3491,8 @@ kernel void cq4_moe_gemv_down_acc( float p = (float)pr[e]; if (p <= 0.0f) continue; float w = p/denom*scaling; - device const uchar* pbase = pk2 + (size_t)e*N*ng*pgb; - device const half* nbase = nm2 + (size_t)e*N*ng; + device const uchar* pbase = pk2 + (size_t)e*estride; + device const half* nbase = (device const half*)((device const uchar*)nm2 + (size_t)e*estride); device const half* cd = code2 + ((size_t)tok*topk_k + slot)*K; threadgroup const float* cb = cbs + (size_t)slot*16u; float acc[2] = {0, 0}; @@ -3500,10 +3500,13 @@ kernel void cq4_moe_gemv_down_acc( uint gg = base/128u, off = base - gg*128u; device const half4* cp = (device const half4*)(cd + gg*128u + off); half4 c0=cp[0], c1=cp[1], c2=cp[2], c3=cp[3]; + device const uchar* pan = pbase + ((size_t)(n0>>2)*ng+gg)*256u + (n0&3u)*4u + (off>>4)*32u; + uint2 Aw = ((device const uint2*)pan)[0]; + uint2 Bw = ((device const uint2*)(pan+16u))[0]; #pragma clang loop unroll(full) for (uint r = 0u; r < 2u; ++r) { - device const ushort4* q = (device const ushort4*)(pbase + ((size_t)(n0+r)*ng+gg)*pgb + off/2u); - acc[r] += (float)nbase[(size_t)(n0+r)*ng+gg]*cq4_dot64(c0, c1, c2, c3, q[0], cb); + float pv = cq4_dot64_ilw(c0, c1, c2, c3, r?Aw.y:Aw.x, r?Bw.y:Bw.x, cb); + acc[r] += (float)nbase[(((size_t)(n0>>2)*ng+gg)<<2)+(n0&3u)+r]*pv; } } #pragma clang loop unroll(full) diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm index d6b79d576..0c4a5f236 100644 --- a/cactus-kernels/src/metal_backend.mm +++ b/cactus-kernels/src/metal_backend.mm @@ -7,6 +7,7 @@ #import #import #include +#include #if TARGET_OS_OSX #include #endif @@ -36,12 +37,12 @@ id psoTsimd = nil; id psoTbatch = nil; id psoMega = nil, psoSwiT = nil, psoCat = nil; - id psoG = nil, psoGmr = nil, psoGmrLow = nil, psoDeqLow = nil; + id psoG = nil, psoGmr = nil, psoGmrLow = nil; id psoActQ = nil, psoG2i8 = nil; - id psoTm = nil, psoGm = nil; - id psoGmma = nil, psoGdense = nil, psoDeq = nil; + id psoTm = nil; + id psoGmma = nil, psoGdense = nil; id psoRotW = nil, psoEmbOW = nil, psoRmsAddRms = nil; - id psoEmbO = nil, psoEmbH = nil; + id psoEmbH = nil; id psoEmbOm = nil, psoEmbHm = nil; id psoCopy=nil, psoBinary=nil, psoScalar=nil, psoUnary=nil, psoRms=nil, psoSwiglu=nil, psoRmsAdd=nil, psoRmsAddScale=nil; id psoCF16F32=nil, psoCF32F16=nil, psoCI8F16=nil, psoCF16I8=nil; @@ -87,12 +88,12 @@ }; psoT=pso("cq4_transform"); psoTsimd=pso("cq4_transform_simd"); psoG=pso("cq4_gemv"); psoTbatch=pso("cq4_transform_batch"); psoGmr=pso("cq4_gemv_mr"); psoMega=pso("cq4_transform_gemv"); psoSwiT=pso("cq4_swiglu_transform"); psoCat=pso("cq4_gemv_cat"); - psoTm=pso("cq4_transform_m"); psoGm=pso("cq4_gemm"); psoGmma=pso("cq4_gemm_mma"); - psoGdense=pso("cq4_gemm_dense_f16"); psoDeq=pso("cq4_dequant_f16"); - psoGmrLow=pso("cq_gemv_mr_lowbit"); psoDeqLow=pso("cq_dequant_lowbit_f16"); + psoTm=pso("cq4_transform_m"); psoGmma=pso("cq4_gemm_mma"); + psoGdense=pso("cq4_gemm_dense_f16"); + psoGmrLow=pso("cq_gemv_mr_lowbit"); psoActQ=pso("cq_act_quant_i8"); psoG2i8=pso("cq2_gemv_i8"); psoRotW=pso("lmhead_rotate_wide"); psoEmbOW=pso("emb_ortho_wide"); psoRmsAddRms=pso("rms_norm_add_rms_f16"); - psoEmbO=pso("emb_ortho"); psoEmbH=pso("emb_hadamard"); + psoEmbH=pso("emb_hadamard"); psoEmbOm=pso("emb_ortho_m"); psoEmbHm=pso("emb_hadamard_m"); psoGather=pso("gather_f16"); psoCopy=pso("copy_bytes"); psoBinary=pso("binary_f16"); psoScalar=pso("scalar_f16"); @@ -134,16 +135,33 @@ psoRopePairRms=pso("rope_pair_rms_f16"); psoRms2AddClip=pso("rms2_add_clip_f16"); psoDeltanet=pso("gated_deltanet_decode_f16"); psoDeltanetPre=pso("gated_deltanet_prefill_f16"); dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; - ok = psoT&&psoG&&psoTm&&psoGm&&psoRotW&&psoEmbO&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 + ok = psoT&&psoG&&psoTm&&psoGmma&&psoRotW&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoArgmax&&psoGather; }} }; MetalCtx& ctx() { static MetalCtx c; return c; } +id owned_shared(size_t len) { + size_t alen = ((len ? len : 1) + 16383) & ~(size_t)16383; + void* p = mmap(nullptr, alen, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); + if (p == MAP_FAILED) + return [ctx().dev newBufferWithLength:alen options:MTLResourceStorageModeShared]; + id b = [ctx().dev newBufferWithBytesNoCopy:p length:alen + options:MTLResourceStorageModeShared + deallocator:^(void* ptr, NSUInteger l){ munmap(ptr, l); }]; + if (!b) { + munmap(p, alen); + return [ctx().dev newBufferWithLength:alen options:MTLResourceStorageModeShared]; + } + return b; +} + inline id buf(const void* p, size_t bytes) { if (!p || bytes == 0) return nil; - return [ctx().dev newBufferWithBytes:p length:bytes options:MTLResourceStorageModeShared]; + id b = owned_shared(bytes); + if (b) std::memcpy([b contents], p, bytes); + return b; } struct ResW { @@ -151,9 +169,10 @@ id rotation=nil; id cb_i8=nil; float cb_scale=1.f; - id wf16=nil; - bool wf16_volatile=false; - uint64_t wf16_tick=0; + size_t packed_off=0, norms_off=0; + size_t cb_off=0, ls_off=0, rs_off=0, pm_off=0, rc_off=0, rot_off=0; + uint32_t il=0; + bool ok=false; }; std::unordered_map g_resident; @@ -169,74 +188,30 @@ static uint64_t resident_key(const CactusQuantMatrix* W) { for (size_t i = pkb - take; i < pkb; ++i) mix(p[i]); return h; } -bool g_wf16_live = false; -size_t g_wf16_bytes = 0; -uint64_t g_wf16_clock = 0; id g_attn_scores = nil; std::unordered_map g_resident_emb; std::mutex g_resident_mu; struct MoESet4 { id pk=nil, nm=nil, rc=nil, ls=nil, rs=nil, pm=nil, cb=nil; - uint32_t K=0, N=0, ng=0; + size_t pk_off=0, nm_off=0, rc_off=0, ls_off=0, rs_off=0, pm_off=0, cb_off=0; + uint32_t K=0, N=0; + uint32_t stride=0; }; struct MoECat4 { MoESet4 w1, w3, w2; bool ok=false; }; std::unordered_map g_moe_cat; -size_t g_moe_bytes = 0; - -void deinterleave_4row(const CactusQuantMatrix* W, - uint8_t* packed_nat, __fp16* norms_nat) { - const uint32_t N=W->N, ng=W->num_groups, gs=W->group_size, pgb=(gs*4u+7u)/8u, NB=N/4; - std::memset(packed_nat, 0, (size_t)N*ng*pgb); - const uint8_t* il=W->packed_indices; - for (uint32_t nb=0; nbnorms[((size_t)nb*ng+g)*4 + r]; - uint8_t* row = packed_nat + ((size_t)n*ng+g)*pgb; - for (uint32_t v=0; v>1] |= (k0&1)?((lo&0xF)<<4):(lo&0xF); - row[k1>>1] |= (k1&1)?((lo>>4)<<4):(lo>>4); - row[k2>>1] |= (k2&1)?((hi&0xF)<<4):(hi&0xF); - row[k3>>1] |= (k3&1)?((hi>>4)<<4):(hi>>4); - } - } - } -} -void deinterleave_4row_lowbit(const CactusQuantMatrix* W, - uint8_t* packed_nat, __fp16* norms_nat) { - const uint32_t N=W->N, ng=W->num_groups, gs=W->group_size, bits=W->bits; - const uint32_t pgb=(gs*bits+7u)/8u; - const uint32_t panel_bytes=gs*4u*bits/8u; - const uint32_t NB=N/4u; - std::memset(packed_nat, 0, (size_t)N*ng*pgb); - const uint8_t* il=W->packed_indices; - for (uint32_t nb=0; nbnorms[((size_t)nb*ng+g)*4u + r]; - uint8_t* row = packed_nat + ((size_t)n*ng+g)*pgb; - for (uint32_t k=0; k>4)*16u; - idx = (ch[((k&15u)>>2)*4u + r] >> (2u*(k&3u))) & 3u; - } else { - const uint8_t* ch = panel + (k>>2)*6u; - uint64_t word=0; std::memcpy(&word, ch, 6); - idx = (uint32_t)((word >> (r*12u + (k&3u)*3u)) & 7u); - } - uint32_t bp = k*bits; - row[bp>>3] |= (uint8_t)(idx << (bp&7u)); - if (bits == 3u && (bp&7u) > 5u) row[(bp>>3)+1u] |= (uint8_t)(idx >> (8u-(bp&7u))); - } - } - } +std::pair, size_t> wrapHostPtr(const void* p, size_t bytes); + +bool bind_component(const void* p, size_t bytes, size_t align, + id __strong& b, size_t& off) { + if (!p || bytes == 0) return true; + if (((uintptr_t)p & (align-1)) != 0) return false; + auto w = wrapHostPtr(p, bytes); + if (!w.first) return false; + b = w.first; + off = w.second; + return true; } ResW& resident(const CactusQuantMatrix* W) { @@ -247,12 +222,12 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, const uint32_t K=W->K, N=W->N, ng=W->num_groups, gs=W->group_size, bits=W->bits; const uint32_t pgb=(gs*bits+7u)/8u; ResW r; - r.codebook = buf(W->codebook, (size_t)(1u<left_signs, gs); - r.rsign = buf(W->right_signs, gs); - r.perm = buf(W->permutation, (size_t)gs*sizeof(uint32_t)); - r.recip = buf(W->input_scale_recip, (size_t)K*sizeof(__fp16)); - if (W->rotation) r.rotation = buf(W->rotation, (size_t)K*K*sizeof(__fp16)); + r.ok = bind_component(W->codebook, (size_t)(1u<left_signs, gs, 1, r.lsign, r.ls_off) + && bind_component(W->right_signs, gs, 1, r.rsign, r.rs_off) + && bind_component(W->permutation, (size_t)gs*sizeof(uint32_t), 4, r.perm, r.pm_off) + && bind_component(W->input_scale_recip, (size_t)K*sizeof(__fp16), 2, r.recip, r.rc_off) + && (!W->rotation || bind_component(W->rotation, (size_t)K*K*sizeof(__fp16), 8, r.rotation, r.rot_off)); if (bits == 2u) { int8_t cbq[4]; float max_abs = 0.f; @@ -268,15 +243,20 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, r.cb_i8 = buf(cbq, 4); r.cb_scale = scale; } - if (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) { - size_t pkb=(size_t)N*ng*pgb, nmb=(size_t)N*ng*sizeof(__fp16); - r.packed = [ctx().dev newBufferWithLength:pkb options:MTLResourceStorageModeShared]; - r.norms = [ctx().dev newBufferWithLength:nmb options:MTLResourceStorageModeShared]; - if (bits == 4u) deinterleave_4row(W, (uint8_t*)r.packed.contents, (__fp16*)r.norms.contents); - else deinterleave_4row_lowbit(W, (uint8_t*)r.packed.contents, (__fp16*)r.norms.contents); - } else { - r.packed = buf(W->packed_indices, (size_t)N*ng*pgb); - r.norms = buf(W->norms, (size_t)N*ng*sizeof(__fp16)); + size_t pkb=(size_t)N*ng*pgb, nmb=(size_t)N*ng*sizeof(__fp16); + const bool interleaved = (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) != 0; + if (((uintptr_t)W->packed_indices & 7u) != 0 || ((uintptr_t)W->norms & 1u) != 0) { + r.ok = false; + } else if (r.ok) { + auto wp = wrapHostPtr(W->packed_indices, pkb); + auto wn = wrapHostPtr(W->norms, nmb); + if (wp.first && wn.first) { + r.packed = wp.first; r.packed_off = wp.second; + r.norms = wn.first; r.norms_off = wn.second; + r.il = interleaved ? 1 : 0; + } else { + r.ok = false; + } } return g_resident.emplace(key, r).first->second; } @@ -290,43 +270,14 @@ void deinterleave_4row_lowbit(const CactusQuantMatrix* W, id g_code_buf = nil; id g_code_buf_m = nil; -inline size_t bucket(size_t b) { return (b + 4095) & ~size_t(4095); } +inline size_t bucket(size_t b) { return (b + 16383) & ~size_t(16383); } static const uint32_t g_mr_rows = 8u * 2u; -static const bool g_predequant=(std::getenv("CACTUS_NO_PREDEQUANT")==nullptr); inline bool tg_mem_ok(size_t bytes) { return bytes <= (size_t)[ctx().dev maxThreadgroupMemoryLength]; } -size_t wf16_budget() { - static const size_t cap = [] { - if (const char* e = std::getenv("CACTUS_WF16_CAP_MB")) return (size_t)atoll(e) << 20; - return (size_t)([ctx().dev recommendedMaxWorkingSetSize] / 4); - }(); - const size_t floor_cap = 64u << 20; - if (g_moe_bytes >= cap) return floor_cap; - size_t left = cap - g_moe_bytes; - return left < floor_cap ? floor_cap : left; -} - -void wf16_evict_for(size_t need, const ResW* keep) { - std::lock_guard lk(g_resident_mu); - const size_t cap = wf16_budget(); - while (g_wf16_bytes + need > cap) { - ResW* victim = nullptr; - for (auto& kv : g_resident) { - ResW& r = kv.second; - if (!r.wf16 || &r == keep) continue; - if (!victim || r.wf16_tick < victim->wf16_tick) victim = &r; - } - if (!victim) break; - g_wf16_bytes -= (size_t)victim->wf16.length; - victim->wf16 = nil; - victim->wf16_volatile = false; - } -} - id ensureEncoder() { if (!g_enc) { if (!g_cmd) g_cmd = [ctx().queue commandBuffer]; @@ -340,14 +291,13 @@ void wf16_evict_for(size_t need, const ResW* keep) { auto it = g_free.find(bk); id b; if (it != g_free.end() && !it->second.empty()) { b = it->second.back(); it->second.pop_back(); } - else b = [ctx().dev newBufferWithLength:bk options:MTLResourceStorageModeShared]; + else b = owned_shared(bk); g_pending.push_back(b); return b; } std::map> g_wrapped; std::unordered_map> g_readonly; -std::unordered_map> g_dequant_cache; std::map, MPSMatrix*> g_mps_mats; std::pair, size_t> wrapHostPtr(const void* p, size_t bytes) { uintptr_t a = (uintptr_t)p, base = a & ~(uintptr_t)16383u; @@ -477,11 +427,8 @@ void cactus_metal_invalidate_host_wraps() { g_readonly.clear(); g_resident.clear(); g_resident_emb.clear(); - g_dequant_cache.clear(); g_mps_mats.clear(); - g_wf16_bytes = 0; g_moe_cat.clear(); - g_moe_bytes = 0; } void cactus_metal_session_flush() { @@ -496,7 +443,7 @@ void cactus_metal_session_flush() { id b = nil; auto it = g_free.find(bk); if (it != g_free.end() && !it->second.empty()) { b = it->second.back(); it->second.pop_back(); } - else b = [ctx().dev newBufferWithLength:bk options:MTLResourceStorageModeShared]; + else b = owned_shared(bk); void* c = [b contents]; g_shared[reinterpret_cast(c)] = b; return c; @@ -527,7 +474,7 @@ static void reclaim_dead_slabs() { } if (!cur || cur->used + need > cur->cap) { size_t cap = 32u << 20; - id b = [ctx().dev newBufferWithLength:cap options:MTLResourceStorageModeShared]; + id b = owned_shared(cap); if (!b) return cactus_metal_alloc_shared(bytes); g_slab = (char*)[b contents]; g_shared[reinterpret_cast(g_slab)] = b; @@ -770,15 +717,12 @@ static inline bool quant_lowbit_eligible(const CactusQuantMatrix* W) { } void cactus_metal_trim_prefill_cache() { - if (!g_wf16_live) return; std::lock_guard lk(g_resident_mu); - for (auto& kv : g_resident) { - if (kv.second.wf16 && !kv.second.wf16_volatile) { - [kv.second.wf16 setPurgeableState:MTLPurgeableStateVolatile]; - kv.second.wf16_volatile = true; - } - } - g_wf16_live = false; + g_mps_mats.clear(); + g_free.clear(); + g_attn_scores = nil; + g_code_buf = nil; + g_code_buf_m = nil; } bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W) { @@ -798,17 +742,18 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu if (!fast && !lowbit) return false; const uint32_t bits = W->bits, pgbw = (gs*bits+7u)/8u; ResW& rw = resident(W); + if (!rw.ok) return false; size_t code_bytes = (size_t)ng*gs*sizeof(__fp16); if (!g_code_buf || (size_t)g_code_buf.length < code_bytes) - g_code_buf = [ctx().dev newBufferWithLength:code_bytes options:MTLResourceStorageModeShared]; + g_code_buf = owned_shared(code_bytes); if (!g_code_buf) return false; ensureEncoder(); bool simdT = (gs==128u && ctx().psoTsimd); [g_enc setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; - setBufAt(lhs, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; - [g_enc setBuffer:rw.lsign offset:0 atIndex:2]; [g_enc setBuffer:rw.rsign offset:0 atIndex:3]; - [g_enc setBuffer:rw.perm offset:0 atIndex:4]; [g_enc setBuffer:g_code_buf offset:0 atIndex:5]; + setBufAt(lhs, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:1]; + [g_enc setBuffer:rw.lsign offset:rw.ls_off atIndex:2]; [g_enc setBuffer:rw.rsign offset:rw.rs_off atIndex:3]; + [g_enc setBuffer:rw.perm offset:rw.pm_off atIndex:4]; [g_enc setBuffer:g_code_buf offset:0 atIndex:5]; [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; if (lowbit && bits == 2u && gs == 128u && ctx().psoActQ && ctx().psoG2i8 && rw.cb_i8) { @@ -821,8 +766,8 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; [g_enc setComputePipelineState:ctx().psoG2i8]; [g_enc setBuffer:actq offset:0 atIndex:0]; [g_enc setBuffer:ascl offset:0 atIndex:1]; - [g_enc setBuffer:rw.packed offset:0 atIndex:2]; [g_enc setBuffer:rw.cb_i8 offset:0 atIndex:3]; - [g_enc setBuffer:rw.norms offset:0 atIndex:4]; + [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:2]; [g_enc setBuffer:rw.cb_i8 offset:0 atIndex:3]; + [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:4]; setBufAt(out, (size_t)N*2, 5); [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&N length:4 atIndex:7]; [g_enc setBytes:&rw.cb_scale length:4 atIndex:8]; @@ -831,12 +776,13 @@ bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQu } bool umr = ctx().psoGmr && (N % g_mr_rows == 0u); [g_enc setComputePipelineState:(lowbit?ctx().psoGmrLow:(umr?ctx().psoGmr:ctx().psoG))]; - [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:0 atIndex:1]; - [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; + [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:1]; + [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:2]; [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:3]; setBufAt(out, (size_t)N*2, 4); [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&pgbw length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; - if (lowbit) [g_enc setBytes:&bits length:4 atIndex:9]; + if (lowbit) { [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc setBytes:&rw.il length:4 atIndex:10]; } + else [g_enc setBytes:&rw.il length:4 atIndex:9]; uint32_t ROWS=8; uint32_t grid = (lowbit || umr) ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; @@ -850,15 +796,16 @@ bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* ResW& r0 = resident(Ws[0]); ResW* rw[3] = { &r0, &r0, &r0 }; for (int b=1;bok) return false; ensureEncoder(); [g_enc setComputePipelineState:ctx().psoTbatch]; setBufAt(x, (size_t)K*2, 0); - [g_enc setBuffer:r0.recip offset:0 atIndex:1]; + [g_enc setBuffer:r0.recip offset:r0.rc_off atIndex:1]; for (int b=0;b<3;b++) { ResW* R = rw[blsign offset:0 atIndex:2+b]; - [g_enc setBuffer:R->rsign offset:0 atIndex:5+b]; - [g_enc setBuffer:R->perm offset:0 atIndex:8+b]; + [g_enc setBuffer:R->lsign offset:R->ls_off atIndex:2+b]; + [g_enc setBuffer:R->rsign offset:R->rs_off atIndex:5+b]; + [g_enc setBuffer:R->perm offset:R->pm_off atIndex:8+b]; setBufAt(codes[bgroup_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; if (!quant_fast_eligible(W)) return false; ResW& rw = resident(W); + if (!rw.ok) return false; ensureEncoder(); bool umr = ctx().psoGmr && (N % g_mr_rows == 0u); [g_enc setComputePipelineState:(umr?ctx().psoGmr:ctx().psoG)]; - setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:0 atIndex:1]; - [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; + setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:1]; + [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:2]; [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:3]; setBufAt(out, (size_t)N*2, 4); [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; + [g_enc setBytes:&rw.il length:4 atIndex:9]; uint32_t ROWS=8; uint32_t grid = umr ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; @@ -896,14 +845,15 @@ bool cactus_metal_encode_transform_gemv(void* out, const void* x, const CactusQu if (gs != 128u || !quant_fast_eligible(W) || (N % 16u) != 0u) return false; if (!cactus_metal_transform_gemv_fits(K)) return false; ResW& rw = resident(W); + if (!rw.ok) return false; ensureEncoder(); struct { uint32_t K, ng, oswi; } U = { K, ng, (uint32_t)(osw ? 1 : 0) }; [g_enc setComputePipelineState:ctx().psoMega]; setBufAt(x, (size_t)K*2, 0); - [g_enc setBuffer:rw.recip offset:0 atIndex:1]; - [g_enc setBuffer:rw.lsign offset:0 atIndex:2]; [g_enc setBuffer:rw.rsign offset:0 atIndex:3]; - [g_enc setBuffer:rw.perm offset:0 atIndex:4]; [g_enc setBuffer:rw.codebook offset:0 atIndex:5]; - [g_enc setBuffer:rw.norms offset:0 atIndex:6]; [g_enc setBuffer:rw.packed offset:0 atIndex:7]; + [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:1]; + [g_enc setBuffer:rw.lsign offset:rw.ls_off atIndex:2]; [g_enc setBuffer:rw.rsign offset:rw.rs_off atIndex:3]; + [g_enc setBuffer:rw.perm offset:rw.pm_off atIndex:4]; [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:5]; + [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:6]; [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:7]; setBufAt(out, (size_t)N*2, 8); if (osw) setBufAt(osw, (size_t)N*2, 9); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:9]; [g_enc setBytes:&U length:sizeof(U) atIndex:10]; @@ -918,13 +868,14 @@ bool cactus_metal_encode_swiglu_transform(void* code, const void* gate, const vo if (!ctx().ok || !ctx().psoSwiT || W->group_size != 128u || !quant_fast_eligible(W)) return false; const uint32_t K=W->K, ng=W->num_groups; ResW& rw = resident(W); + if (!rw.ok) return false; ensureEncoder(); float s = scale; [g_enc setComputePipelineState:ctx().psoSwiT]; setBufAt(gate, (size_t)K*2, 0); setBufAt(up, (size_t)K*2, 1); - [g_enc setBuffer:rw.recip offset:0 atIndex:2]; - [g_enc setBuffer:rw.lsign offset:0 atIndex:3]; [g_enc setBuffer:rw.rsign offset:0 atIndex:4]; - [g_enc setBuffer:rw.perm offset:0 atIndex:5]; + [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:2]; + [g_enc setBuffer:rw.lsign offset:rw.ls_off atIndex:3]; [g_enc setBuffer:rw.rsign offset:rw.rs_off atIndex:4]; + [g_enc setBuffer:rw.perm offset:rw.pm_off atIndex:5]; setBufAt(code, (size_t)K*2, 6); [g_enc setBytes:&s length:4 atIndex:7]; [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; @@ -944,15 +895,16 @@ bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, } ResW* rw[3]; rw[0]=&resident(Ws[0]); rw[1]=rw[0]; rw[2]=rw[0]; for (int i=1;iok) return false; ensureEncoder(); struct { uint32_t ng, N0, N1, N2; } U = { ng, Ns[0], Ns[1], Ns[2] }; [g_enc setComputePipelineState:ctx().psoCat]; for (int i=0;i<3;i++){ int j = ipacked offset:0 atIndex:3+i]; - [g_enc setBuffer:rw[j]->codebook offset:0 atIndex:6+i]; - [g_enc setBuffer:rw[j]->norms offset:0 atIndex:9+i]; + [g_enc setBuffer:rw[j]->packed offset:rw[j]->packed_off atIndex:3+i]; + [g_enc setBuffer:rw[j]->codebook offset:rw[j]->cb_off atIndex:6+i]; + [g_enc setBuffer:rw[j]->norms offset:rw[j]->norms_off atIndex:9+i]; setBufAt(outs[j], (size_t)Ns[j]*2, 12+i); } [g_enc setBytes:&U length:sizeof(U) atIndex:15]; @@ -964,34 +916,85 @@ bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { if (!ctx().ok) return false; if (!quant_fast_eligible(W) && !quant_lowbit_eligible(W)) return false; - resident(W); - return true; + return resident(W).ok; } namespace { -void moe_release_source(const CactusQuantMatrix* W) { - const uint32_t pgb = (W->group_size*W->bits + 7u)/8u; +bool moe_component_offsets(const CactusQuantMatrix& W, uintptr_t& lo, uintptr_t& hi, size_t offs[7]) { + const uintptr_t pg = 16384; struct Span { const void* p; size_t len; }; Span spans[7] = { - { W->packed_indices, (size_t)W->N*W->num_groups*pgb }, - { W->norms, (size_t)W->N*W->num_groups*2 }, - { W->input_scale_recip, (size_t)W->K*2 }, - { W->left_signs, (size_t)W->group_size }, - { W->right_signs, (size_t)W->group_size }, - { W->permutation, (size_t)W->group_size*4 }, - { W->codebook, (size_t)(1u<bits)*2 }, + { W.packed_indices, (size_t)W.N*W.num_groups*64u }, + { W.norms, (size_t)W.N*W.num_groups*2 }, + { W.input_scale_recip, (size_t)W.K*2 }, + { W.left_signs, 128 }, + { W.right_signs, 128 }, + { W.permutation, 512 }, + { W.codebook, 32 }, }; - uintptr_t lo = UINTPTR_MAX, hi = 0; + uintptr_t mn = UINTPTR_MAX, mx = 0; for (const auto& s : spans) { - if (!s.p) continue; + if (!s.p) return false; uintptr_t a = (uintptr_t)s.p; - if (a < lo) lo = a; - if (a + s.len > hi) hi = a + s.len; + if (a < mn) mn = a; + if (a + s.len > mx) mx = a + s.len; } - if (hi <= lo) return; - const uintptr_t pg = 16384; - uintptr_t start = (lo + pg - 1) & ~(pg - 1), end = hi & ~(pg - 1); - if (end > start) madvise((void*)start, end - start, MADV_DONTNEED); + lo = mn & ~(pg - 1); + hi = (mx + pg - 1) & ~(pg - 1); + for (int i = 0; i < 7; i++) offs[i] = (uintptr_t)spans[i].p - lo; + return true; +} + +bool moe_build_set4_zerocopy(MoESet4& S, const CactusQuantMatrix* Ws, uint32_t count) { + if (count == 0) return false; + uintptr_t lo0, hi0; + size_t offs[7]; + if (!moe_component_offsets(Ws[0], lo0, hi0, offs)) return false; + const size_t slot = hi0 - lo0; + if (slot > 0xFFFFFFFFull) return false; + if ((offs[0] & 7u) || (offs[1] & 1u) || (offs[2] & 1u) || (offs[6] & 1u) || (offs[5] & 3u)) return false; + for (uint32_t e = 1; e < count; ++e) { + uintptr_t lo, hi; + size_t o[7]; + if (!moe_component_offsets(Ws[e], lo, hi, o)) return false; + if (hi - lo != slot) return false; + for (int i = 0; i < 7; i++) if (o[i] != offs[i]) return false; + } + const size_t total = slot * count; + void* arena = mmap(nullptr, total, PROT_NONE, MAP_ANON | MAP_PRIVATE, -1, 0); + if (arena == MAP_FAILED) return false; + for (uint32_t e = 0; e < count; ++e) { + uintptr_t lo, hi; + size_t o[7]; + moe_component_offsets(Ws[e], lo, hi, o); + vm_address_t dst = (vm_address_t)((uintptr_t)arena + (size_t)e*slot); + vm_prot_t curp = 0, maxp = 0; + kern_return_t kr = vm_remap(mach_task_self(), &dst, slot, 0, + VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, mach_task_self(), + (vm_address_t)lo, FALSE, &curp, &maxp, VM_INHERIT_NONE); + if (kr != KERN_SUCCESS || dst != (vm_address_t)((uintptr_t)arena + (size_t)e*slot) + || !(curp & VM_PROT_READ)) { + munmap(arena, total); + return false; + } + } + id ab = [ctx().dev newBufferWithBytesNoCopy:arena length:total + options:MTLResourceStorageModeShared deallocator:nil]; + if (!ab) { munmap(arena, total); return false; } + CactusThreading::parallel_for(count, CactusThreading::ParallelConfig(1, 1), + [&](size_t start, size_t end) { + volatile uint64_t acc = 0; + for (size_t e = start; e < end; ++e) + for (size_t o = 0; o < slot; o += 16384) + acc += *((const volatile uint8_t*)arena + e*slot + o); + (void)acc; + }); + S.K=Ws[0].K; S.N=Ws[0].N; + S.pk=S.nm=S.rc=S.ls=S.rs=S.pm=S.cb=ab; + S.pk_off=offs[0]; S.nm_off=offs[1]; S.rc_off=offs[2]; S.ls_off=offs[3]; + S.rs_off=offs[4]; S.pm_off=offs[5]; S.cb_off=offs[6]; + S.stride = (uint32_t)slot; + return true; } bool moe_build_set4(MoESet4& S, const CactusQuantMatrix* Ws, uint32_t count) { @@ -1001,32 +1004,7 @@ bool moe_build_set4(MoESet4& S, const CactusQuantMatrix* Ws, uint32_t count) { const CactusQuantMatrix* W = &Ws[e]; if (!quant_fast_eligible(W) || W->K != K || W->N != N || W->num_groups != ng) return false; } - const uint32_t pgb = 64u; - const size_t pk_stride = (size_t)N*ng*pgb, nm_stride = (size_t)N*ng; - S.K=K; S.N=N; S.ng=ng; - S.pk=[ctx().dev newBufferWithLength:pk_stride*count options:MTLResourceStorageModeShared]; - S.nm=[ctx().dev newBufferWithLength:nm_stride*count*2 options:MTLResourceStorageModeShared]; - S.rc=[ctx().dev newBufferWithLength:(size_t)K*count*2 options:MTLResourceStorageModeShared]; - S.ls=[ctx().dev newBufferWithLength:(size_t)128*count options:MTLResourceStorageModeShared]; - S.rs=[ctx().dev newBufferWithLength:(size_t)128*count options:MTLResourceStorageModeShared]; - S.pm=[ctx().dev newBufferWithLength:(size_t)128*count*4 options:MTLResourceStorageModeShared]; - S.cb=[ctx().dev newBufferWithLength:(size_t)16*count*2 options:MTLResourceStorageModeShared]; - if (!S.pk||!S.nm||!S.rc||!S.ls||!S.rs||!S.pm||!S.cb) return false; - CactusThreading::parallel_for(count, CactusThreading::ParallelConfig(1, 1), - [&](size_t start, size_t end) { - for (size_t e = start; e < end; ++e) { - const CactusQuantMatrix* W = &Ws[e]; - deinterleave_4row(W, (uint8_t*)S.pk.contents + pk_stride*e, - (__fp16*)S.nm.contents + nm_stride*e); - std::memcpy((__fp16*)S.rc.contents + (size_t)K*e, W->input_scale_recip, (size_t)K*2); - std::memcpy((char*)S.ls.contents + 128*e, W->left_signs, 128); - std::memcpy((char*)S.rs.contents + 128*e, W->right_signs, 128); - std::memcpy((uint32_t*)S.pm.contents + 128*e, W->permutation, 128*4); - std::memcpy((__fp16*)S.cb.contents + 16*e, W->codebook, 16*2); - moe_release_source(W); - } - }); - return true; + return moe_build_set4_zerocopy(S, Ws, count); } } @@ -1054,11 +1032,6 @@ bool cactus_metal_moe_cq4_build(const CactusQuantMatrix* w1s, const CactusQuantM if (!cat.ok) cat = MoECat4{}; std::lock_guard lk(g_resident_mu); auto ins = g_moe_cat.emplace(key, cat); - if (ins.second && cat.ok) { - for (const MoESet4* st : {&cat.w1, &cat.w3, &cat.w2}) - for (id b : {st->pk, st->nm, st->rc, st->ls, st->rs, st->pm, st->cb}) - if (b) g_moe_bytes += (size_t)b.length; - } return ins.first->second.ok; } @@ -1086,18 +1059,20 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void [g_enc setComputePipelineState:ctx().psoMoeT2]; setBufAt(hidden, (size_t)tokens*K1*2, 0); setBufAt(topk, slots*sizeof(float), 1); - [g_enc setBuffer:C.w1.rc offset:0 atIndex:2]; - [g_enc setBuffer:C.w1.ls offset:0 atIndex:3]; - [g_enc setBuffer:C.w1.rs offset:0 atIndex:4]; - [g_enc setBuffer:C.w1.pm offset:0 atIndex:5]; - [g_enc setBuffer:C.w3.rc offset:0 atIndex:6]; - [g_enc setBuffer:C.w3.ls offset:0 atIndex:7]; - [g_enc setBuffer:C.w3.rs offset:0 atIndex:8]; - [g_enc setBuffer:C.w3.pm offset:0 atIndex:9]; + [g_enc setBuffer:C.w1.rc offset:C.w1.rc_off atIndex:2]; + [g_enc setBuffer:C.w1.ls offset:C.w1.ls_off atIndex:3]; + [g_enc setBuffer:C.w1.rs offset:C.w1.rs_off atIndex:4]; + [g_enc setBuffer:C.w1.pm offset:C.w1.pm_off atIndex:5]; + [g_enc setBuffer:C.w3.rc offset:C.w3.rc_off atIndex:6]; + [g_enc setBuffer:C.w3.ls offset:C.w3.ls_off atIndex:7]; + [g_enc setBuffer:C.w3.rs offset:C.w3.rs_off atIndex:8]; + [g_enc setBuffer:C.w3.pm offset:C.w3.pm_off atIndex:9]; [g_enc setBuffer:code1 offset:0 atIndex:10]; [g_enc setBuffer:code3 offset:0 atIndex:11]; uint32_t Kc=K1, tk=top_k; [g_enc setBytes:&Kc length:4 atIndex:12]; [g_enc setBytes:&tk length:4 atIndex:13]; + [g_enc setBytes:&C.w1.stride length:4 atIndex:14]; + [g_enc setBytes:&C.w3.stride length:4 atIndex:15]; [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(K1/128u, 2u*top_k, tokens) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; @@ -1106,12 +1081,14 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void [g_enc setComputePipelineState:ctx().psoMoeUp]; [g_enc setBuffer:code1 offset:0 atIndex:0]; [g_enc setBuffer:code3 offset:0 atIndex:1]; setBufAt(topk, slots*sizeof(float), 2); - [g_enc setBuffer:C.w1.pk offset:0 atIndex:3]; [g_enc setBuffer:C.w1.nm offset:0 atIndex:4]; [g_enc setBuffer:C.w1.cb offset:0 atIndex:5]; - [g_enc setBuffer:C.w3.pk offset:0 atIndex:6]; [g_enc setBuffer:C.w3.nm offset:0 atIndex:7]; [g_enc setBuffer:C.w3.cb offset:0 atIndex:8]; + [g_enc setBuffer:C.w1.pk offset:C.w1.pk_off atIndex:3]; [g_enc setBuffer:C.w1.nm offset:C.w1.nm_off atIndex:4]; [g_enc setBuffer:C.w1.cb offset:C.w1.cb_off atIndex:5]; + [g_enc setBuffer:C.w3.pk offset:C.w3.pk_off atIndex:6]; [g_enc setBuffer:C.w3.nm offset:C.w3.nm_off atIndex:7]; [g_enc setBuffer:C.w3.cb offset:C.w3.cb_off atIndex:8]; [g_enc setBuffer:gu offset:0 atIndex:9]; uint32_t Kc=K1, Nc=N1, a=act, tk=top_k; [g_enc setBytes:&Kc length:4 atIndex:10]; [g_enc setBytes:&Nc length:4 atIndex:11]; [g_enc setBytes:&a length:4 atIndex:12]; [g_enc setBytes:&tk length:4 atIndex:13]; + [g_enc setBytes:&C.w1.stride length:4 atIndex:14]; + [g_enc setBytes:&C.w3.stride length:4 atIndex:15]; [g_enc dispatchThreadgroups:MTLSizeMake((N1+31u)/32u, top_k, tokens) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; } @@ -1119,15 +1096,16 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void [g_enc setComputePipelineState:ctx().psoMoeT]; [g_enc setBuffer:gu offset:0 atIndex:0]; setBufAt(topk, slots*sizeof(float), 1); - [g_enc setBuffer:C.w2.rc offset:0 atIndex:2]; - [g_enc setBuffer:C.w2.ls offset:0 atIndex:3]; - [g_enc setBuffer:C.w2.rs offset:0 atIndex:4]; - [g_enc setBuffer:C.w2.pm offset:0 atIndex:5]; + [g_enc setBuffer:C.w2.rc offset:C.w2.rc_off atIndex:2]; + [g_enc setBuffer:C.w2.ls offset:C.w2.ls_off atIndex:3]; + [g_enc setBuffer:C.w2.rs offset:C.w2.rs_off atIndex:4]; + [g_enc setBuffer:C.w2.pm offset:C.w2.pm_off atIndex:5]; [g_enc setBuffer:code2 offset:0 atIndex:6]; uint32_t Kc=K2, kv=N1, xz=top_k*N1, xy=N1, tk=top_k; [g_enc setBytes:&Kc length:4 atIndex:7]; [g_enc setBytes:&kv length:4 atIndex:8]; [g_enc setBytes:&xz length:4 atIndex:9]; [g_enc setBytes:&xy length:4 atIndex:10]; [g_enc setBytes:&tk length:4 atIndex:11]; + [g_enc setBytes:&C.w2.stride length:4 atIndex:12]; [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(K2/128u, top_k, tokens) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; @@ -1137,13 +1115,14 @@ bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void [g_enc setBuffer:code2 offset:0 atIndex:0]; setBufAt(topk, slots*sizeof(float), 1); setBufAt(probs, (size_t)tokens*num_experts*2, 2); - [g_enc setBuffer:C.w2.pk offset:0 atIndex:3]; [g_enc setBuffer:C.w2.nm offset:0 atIndex:4]; [g_enc setBuffer:C.w2.cb offset:0 atIndex:5]; + [g_enc setBuffer:C.w2.pk offset:C.w2.pk_off atIndex:3]; [g_enc setBuffer:C.w2.nm offset:C.w2.nm_off atIndex:4]; [g_enc setBuffer:C.w2.cb offset:C.w2.cb_off atIndex:5]; setBufAt(out, (size_t)tokens*N2*2, 6); uint32_t Kc=K2, Nc=N2, tk=top_k, nz=normalize, Ec=num_experts; [g_enc setBytes:&Kc length:4 atIndex:7]; [g_enc setBytes:&Nc length:4 atIndex:8]; [g_enc setBytes:&tk length:4 atIndex:9]; [g_enc setBytes:&nz length:4 atIndex:10]; [g_enc setBytes:&eps length:4 atIndex:11]; [g_enc setBytes:&scaling length:4 atIndex:12]; [g_enc setBytes:&Ec length:4 atIndex:13]; + [g_enc setBytes:&C.w2.stride length:4 atIndex:14]; [g_enc setThreadgroupMemoryLength:top_k*16*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake((N2+15u)/16u, 1, tokens) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; @@ -1305,75 +1284,30 @@ bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const Cactus const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; const uint32_t bits=W->bits, pgb=(gs*bits+7u)/8u; const bool fast = ctx().ok && quant_fast_eligible(W); - const bool lowbit = !fast && ctx().ok && quant_lowbit_eligible(W) && - g_predequant && ctx().psoGdense && ctx().psoDeqLow; - if (!fast && !lowbit) return false; + if (!fast || !ctx().psoGmma) return false; ResW& rw = resident(W); + if (!rw.ok) return false; size_t code_bytes = (size_t)M*ng*gs*sizeof(__fp16); if (!g_code_buf_m || (size_t)g_code_buf_m.length < code_bytes) - g_code_buf_m = [ctx().dev newBufferWithLength:code_bytes options:MTLResourceStorageModeShared]; + g_code_buf_m = owned_shared(code_bytes); if (!g_code_buf_m) return false; ensureEncoder(); [g_enc setComputePipelineState:ctx().psoTm]; - setBufAt(lhs, (size_t)M*K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; - [g_enc setBuffer:rw.lsign offset:0 atIndex:2]; [g_enc setBuffer:rw.rsign offset:0 atIndex:3]; - [g_enc setBuffer:rw.perm offset:0 atIndex:4]; [g_enc setBuffer:g_code_buf_m offset:0 atIndex:5]; + setBufAt(lhs, (size_t)M*K*2, 0); [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:1]; + [g_enc setBuffer:rw.lsign offset:rw.ls_off atIndex:2]; [g_enc setBuffer:rw.rsign offset:rw.rs_off atIndex:3]; + [g_enc setBuffer:rw.perm offset:rw.pm_off atIndex:4]; [g_enc setBuffer:g_code_buf_m offset:0 atIndex:5]; [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setBytes:&K length:4 atIndex:7]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; - bool dense = (g_predequant && ctx().psoGdense && ctx().psoDeq) || lowbit; - bool need_dequant = false; - if (dense) { - if (!rw.wf16) { - size_t wbytes = (size_t)N*K*2; - wf16_evict_for(wbytes, &rw); - rw.wf16 = [ctx().dev newBufferWithLength:wbytes options:MTLResourceStorageModeShared]; - if (rw.wf16) g_wf16_bytes += (size_t)rw.wf16.length; - need_dequant = true; - } else if (rw.wf16_volatile) { - need_dequant = [rw.wf16 setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty; - } - if (!rw.wf16) { - if (lowbit) return false; - dense = false; - } - } - if (dense) { - rw.wf16_volatile = false; - rw.wf16_tick = ++g_wf16_clock; - g_wf16_live = true; - if (need_dequant) { - [g_enc setComputePipelineState:(lowbit?ctx().psoDeqLow:ctx().psoDeq)]; - [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; - [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.wf16 offset:0 atIndex:3]; - [g_enc setBytes:&gs length:4 atIndex:4]; [g_enc setBytes:&ng length:4 atIndex:5]; - [g_enc setBytes:&pgb length:4 atIndex:6]; [g_enc setBytes:&N length:4 atIndex:7]; - if (lowbit) [g_enc setBytes:&bits length:4 atIndex:8]; - [g_enc dispatchThreads:MTLSizeMake((size_t)N*ng,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; - } - [g_enc setComputePipelineState:ctx().psoGdense]; - [g_enc setBuffer:g_code_buf_m offset:0 atIndex:0]; [g_enc setBuffer:rw.wf16 offset:0 atIndex:1]; - setBufAt(out, (size_t)M*N*2, 2); - [g_enc setBytes:&K length:4 atIndex:3]; [g_enc setBytes:&N length:4 atIndex:4]; [g_enc setBytes:&M length:4 atIndex:5]; - [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; - return true; - } - - bool mma = (ctx().psoGmma != nil); - [g_enc setComputePipelineState:(mma?ctx().psoGmma:ctx().psoGm)]; - [g_enc setBuffer:g_code_buf_m offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:0 atIndex:1]; - [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; + [g_enc setComputePipelineState:ctx().psoGmma]; + [g_enc setBuffer:g_code_buf_m offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:1]; + [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:2]; [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:3]; setBufAt(out, (size_t)M*N*2, 4); [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; [g_enc setBytes:&M length:4 atIndex:9]; - if (mma) { - [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; - } else { - uint32_t ROWS=8, MT=16; - [g_enc dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,(M+MT-1)/MT,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; - } + [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; return true; } @@ -1387,24 +1321,25 @@ bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* co if (lowbit && (!ctx().psoGmrLow || (N % 16) != 0)) return false; if (ng != 1 || gs != K || (N % 4) != 0) return false; ResW& rw = resident(W); - if (!rw.rotation || !rw.packed) return false; + if (!rw.ok || !rw.rotation || !rw.packed) return false; ensureEncoder(); [g_enc setComputePipelineState:ctx().psoRotW]; - setBufAt(act, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:0 atIndex:1]; - [g_enc setBuffer:rw.rotation offset:0 atIndex:2]; setBufAt(code, (size_t)K*2, 3); + setBufAt(act, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:1]; + [g_enc setBuffer:rw.rotation offset:rw.rot_off atIndex:2]; setBufAt(code, (size_t)K*2, 3); [g_enc setBytes:&K length:4 atIndex:4]; [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake((K+31)/32,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; bool umr = ctx().psoGmr && (N % g_mr_rows == 0u); [g_enc setComputePipelineState:(lowbit?ctx().psoGmrLow:(umr?ctx().psoGmr:ctx().psoG))]; - setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:0 atIndex:1]; - [g_enc setBuffer:rw.codebook offset:0 atIndex:2]; [g_enc setBuffer:rw.norms offset:0 atIndex:3]; + setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:1]; + [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:2]; [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:3]; setBufAt(out, (size_t)N*2, 4); [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; - if (lowbit) [g_enc setBytes:&bits length:4 atIndex:9]; + if (lowbit) { [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc setBytes:&rw.il length:4 atIndex:10]; } + else [g_enc setBytes:&rw.il length:4 atIndex:9]; uint32_t ROWS=8; uint32_t grid = (lowbit || umr) ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; @@ -1417,43 +1352,46 @@ bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQu if (ng != 1 || gs != K) return false; if (!tg_mem_ok((size_t)K*sizeof(float))) return false; if (bits != 4 && !(ctx().psoEmbOW && (K % 8u) == 0u)) return false; + if (!ctx().psoEmbOW || (K % 8u) != 0u) return false; ResW& rw = resident(W); - if (!rw.packed || !rw.rotation || !rw.codebook || !rw.norms || !rw.recip) return false; + if (!rw.ok || !rw.packed || !rw.rotation || !rw.codebook || !rw.norms || !rw.recip) return false; ensureEncoder(); - if (!(ctx().psoEmbOW && (K % 8u) == 0u) && scale != 1.0f) return false; - if (ctx().psoEmbOW && (K % 8u) == 0u) { + { [g_enc setComputePipelineState:ctx().psoEmbOW]; - [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; - [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.recip offset:0 atIndex:3]; - [g_enc setBuffer:rw.rotation offset:0 atIndex:4]; setBufAt(out, (size_t)K*2, 5); + [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:0]; [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:1]; + [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:2]; [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:3]; + [g_enc setBuffer:rw.rotation offset:rw.rot_off atIndex:4]; setBufAt(out, (size_t)K*2, 5); [g_enc setBytes:&K length:4 atIndex:6]; [g_enc setBytes:&row length:4 atIndex:7]; [g_enc setBytes:&scale length:4 atIndex:8]; [g_enc setBytes:&bits length:4 atIndex:9]; + [g_enc setBytes:&rw.il length:4 atIndex:10]; [g_enc setThreadgroupMemoryLength:(size_t)K*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(K/8u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } - [g_enc setComputePipelineState:ctx().psoEmbO]; - [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; - [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.recip offset:0 atIndex:3]; - [g_enc setBuffer:rw.rotation offset:0 atIndex:4]; setBufAt(out, (size_t)K*2, 5); - [g_enc setBytes:&K length:4 atIndex:6]; [g_enc setBytes:&row length:4 atIndex:7]; - uint32_t T = K>512u ? 512u : K; - [g_enc setThreadgroupMemoryLength:(size_t)K*sizeof(float) atIndex:0]; - [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; - return true; } ResW& resident_emb_meta(const CactusQuantMatrix* W) { std::lock_guard lk(g_resident_mu); auto it = g_resident_emb.find(W->packed_indices); if (it != g_resident_emb.end()) return it->second; - const uint32_t K=W->K, gs=W->group_size; + const uint32_t K=W->K, gs=W->group_size, ng=W->num_groups, bits=W->bits; + const uint32_t pgb=(gs*bits+7u)/8u; ResW r; - r.codebook = buf(W->codebook, (size_t)(1u<bits)*sizeof(__fp16)); - r.lsign = buf(W->left_signs, gs); - r.rsign = buf(W->right_signs, gs); - r.perm = buf(W->permutation, (size_t)gs*sizeof(uint32_t)); - r.recip = buf(W->input_scale_recip, (size_t)K*sizeof(__fp16)); + bind_component(W->codebook, (size_t)(1u<bits)*sizeof(__fp16), 2, r.codebook, r.cb_off); + bind_component(W->left_signs, gs, 1, r.lsign, r.ls_off); + bind_component(W->right_signs, gs, 1, r.rsign, r.rs_off); + bind_component(W->permutation, (size_t)gs*sizeof(uint32_t), 4, r.perm, r.pm_off); + bind_component(W->input_scale_recip, (size_t)K*sizeof(__fp16), 2, r.recip, r.rc_off); + if (W->N && ng) { + auto wp = wrapHostPtr(W->packed_indices, (size_t)W->N*ng*pgb); + if (wp.first && ((uintptr_t)W->norms & 1u) == 0) { + auto wn = wrapHostPtr(W->norms, (size_t)W->N*ng*sizeof(__fp16)); + if (wn.first) { + r.packed = wp.first; r.packed_off = wp.second; + r.norms = wn.first; r.norms_off = wn.second; + } + } + } return g_resident_emb.emplace(W->packed_indices, r).first->second; } @@ -1467,15 +1405,14 @@ bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const Cactu if (!rm.codebook || !rm.recip || !rm.lsign || !rm.rsign || !rm.perm) return false; ensureEncoder(); size_t row_bytes = (size_t)ng*pgb; - id prow = recycled(row_bytes); - std::memcpy([prow contents], (const uint8_t*)W->packed_indices + (size_t)row*ng*pgb, row_bytes); - id nrow = recycled((size_t)ng*sizeof(__fp16)); - std::memcpy([nrow contents], (const __fp16*)W->norms + (size_t)row*ng, (size_t)ng*sizeof(__fp16)); + if (!rm.packed || !rm.norms) return false; [g_enc setComputePipelineState:ctx().psoEmbH]; - [g_enc setBuffer:prow offset:0 atIndex:0]; [g_enc setBuffer:rm.codebook offset:0 atIndex:1]; - [g_enc setBuffer:nrow offset:0 atIndex:2]; [g_enc setBuffer:rm.recip offset:0 atIndex:3]; - [g_enc setBuffer:rm.lsign offset:0 atIndex:4]; [g_enc setBuffer:rm.rsign offset:0 atIndex:5]; - [g_enc setBuffer:rm.perm offset:0 atIndex:6]; setBufAt(out, (size_t)K*2, 7); + [g_enc setBuffer:rm.packed offset:rm.packed_off + (size_t)row*row_bytes atIndex:0]; + [g_enc setBuffer:rm.norms offset:rm.norms_off + (size_t)row*ng*sizeof(__fp16) atIndex:2]; + [g_enc setBuffer:rm.codebook offset:rm.cb_off atIndex:1]; + [g_enc setBuffer:rm.recip offset:rm.rc_off atIndex:3]; + [g_enc setBuffer:rm.lsign offset:rm.ls_off atIndex:4]; [g_enc setBuffer:rm.rsign offset:rm.rs_off atIndex:5]; + [g_enc setBuffer:rm.perm offset:rm.pm_off atIndex:6]; setBufAt(out, (size_t)K*2, 7); [g_enc setBytes:&gs length:4 atIndex:8]; [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc setThreadgroupMemoryLength:(size_t)gs*sizeof(float) atIndex:0]; [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(gs,1,1)]; @@ -1504,11 +1441,12 @@ bool cactus_metal_encode_embedding_ortho_m(void* out, const CactusQuantMatrix* W id rb = recycled((size_t)M*sizeof(uint32_t)); std::memcpy([rb contents], rows, (size_t)M*sizeof(uint32_t)); [g_enc setComputePipelineState:ctx().psoEmbOm]; - [g_enc setBuffer:rw.packed offset:0 atIndex:0]; [g_enc setBuffer:rw.codebook offset:0 atIndex:1]; - [g_enc setBuffer:rw.norms offset:0 atIndex:2]; [g_enc setBuffer:rw.recip offset:0 atIndex:3]; - [g_enc setBuffer:rw.rotation offset:0 atIndex:4]; [g_enc setBuffer:rb offset:0 atIndex:5]; + [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:0]; [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:1]; + [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:2]; [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:3]; + [g_enc setBuffer:rw.rotation offset:rw.rot_off atIndex:4]; [g_enc setBuffer:rb offset:0 atIndex:5]; setBufAt(out, (size_t)M*K*2, 6); [g_enc setBytes:&K length:4 atIndex:7]; [g_enc setBytes:&M length:4 atIndex:8]; [g_enc setBytes:&bits length:4 atIndex:9]; + [g_enc setBytes:&rw.il length:4 atIndex:10]; [g_enc dispatchThreadgroups:MTLSizeMake(((K+15)/16)*((M+15)/16),1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; return true; } @@ -1530,10 +1468,10 @@ bool cactus_metal_encode_embedding_hadamard_m(void* out, const CactusQuantMatrix std::memcpy((__fp16*)[nm contents]+(size_t)m*ng, (const __fp16*)W->norms+(size_t)rows[m]*ng, (size_t)ng*sizeof(__fp16)); } [g_enc setComputePipelineState:ctx().psoEmbHm]; - [g_enc setBuffer:pk offset:0 atIndex:0]; [g_enc setBuffer:rm.codebook offset:0 atIndex:1]; - [g_enc setBuffer:nm offset:0 atIndex:2]; [g_enc setBuffer:rm.recip offset:0 atIndex:3]; - [g_enc setBuffer:rm.lsign offset:0 atIndex:4]; [g_enc setBuffer:rm.rsign offset:0 atIndex:5]; - [g_enc setBuffer:rm.perm offset:0 atIndex:6]; setBufAt(out, (size_t)M*K*2, 7); + [g_enc setBuffer:pk offset:0 atIndex:0]; [g_enc setBuffer:rm.codebook offset:rm.cb_off atIndex:1]; + [g_enc setBuffer:nm offset:0 atIndex:2]; [g_enc setBuffer:rm.recip offset:rm.rc_off atIndex:3]; + [g_enc setBuffer:rm.lsign offset:rm.ls_off atIndex:4]; [g_enc setBuffer:rm.rsign offset:rm.rs_off atIndex:5]; + [g_enc setBuffer:rm.perm offset:rm.pm_off atIndex:6]; setBufAt(out, (size_t)M*K*2, 7); [g_enc setBytes:&gs length:4 atIndex:8]; [g_enc setBytes:&ng length:4 atIndex:9]; [g_enc setBytes:&K length:4 atIndex:10]; [g_enc setBytes:&bits length:4 atIndex:11]; [g_enc setThreadgroupMemoryLength:(size_t)gs*sizeof(float) atIndex:0]; @@ -1808,32 +1746,28 @@ bool cactus_metal_encode_conv1d_k3(void* out, const void* x, const void* w, int const void* w_scales, uint32_t w_gs, uint32_t Cin, uint32_t L, uint32_t Cout, uint32_t Lout, uint32_t stride) { if (!ctx().ok || !ctx().psoConv1dK3) return false; - const void* wf16 = w; + ensureEncoder(); + id dq = nil; if (w_int8) { - std::lock_guard lk(g_resident_mu); - auto it = g_dequant_cache.find(w); - if (it == g_dequant_cache.end()) { - size_t rows = Cout, cols = (size_t)Cin*3; - id dq = [ctx().dev newBufferWithLength:rows*cols*2 options:MTLResourceStorageModeShared]; - if (!dq) return false; - __fp16* dst = (__fp16*)[dq contents]; - const int8_t* src = (const int8_t*)w; - if (w_gs == 0 || !w_scales) { - for (size_t i = 0; i < rows*cols; ++i) dst[i] = (__fp16)src[i]; - } else { - const __fp16* scales = (const __fp16*)w_scales; - size_t ngr = cols / w_gs; - for (size_t r = 0; r < rows; ++r) - for (size_t c = 0; c < cols; ++c) - dst[r*cols+c] = (__fp16)((float)src[r*cols+c] * (float)scales[r*ngr + c/w_gs]); - } - it = g_dequant_cache.emplace(w, dq).first; + size_t rows = Cout, cols = (size_t)Cin*3; + dq = recycled(rows*cols*2); + if (!dq) return false; + __fp16* dst = (__fp16*)[dq contents]; + const int8_t* src = (const int8_t*)w; + if (w_gs == 0 || !w_scales) { + for (size_t i = 0; i < rows*cols; ++i) dst[i] = (__fp16)src[i]; + } else { + const __fp16* scales = (const __fp16*)w_scales; + size_t ngr = cols / w_gs; + for (size_t r = 0; r < rows; ++r) + for (size_t c = 0; c < cols; ++c) + dst[r*cols+c] = (__fp16)((float)src[r*cols+c] * (float)scales[r*ngr + c/w_gs]); } - wf16 = [it->second contents]; } - ensureEncoder(); [g_enc setComputePipelineState:ctx().psoConv1dK3]; - setBufAt(x, (size_t)Cin*L*2, 0); setBufAt(wf16, (size_t)Cout*Cin*3*2, 1); setBufAt(out, (size_t)Cout*Lout*2, 2); + setBufAt(x, (size_t)Cin*L*2, 0); + if (dq) [g_enc setBuffer:dq offset:0 atIndex:1]; else setBufAt(w, (size_t)Cout*Cin*3*2, 1); + setBufAt(out, (size_t)Cout*Lout*2, 2); [g_enc setBytes:&Cin length:4 atIndex:3]; [g_enc setBytes:&L length:4 atIndex:4]; [g_enc setBytes:&Lout length:4 atIndex:5]; [g_enc setBytes:&stride length:4 atIndex:6]; [g_enc dispatchThreads:MTLSizeMake(Lout,Cout,1) threadsPerThreadgroup:MTLSizeMake(64,4,1)]; @@ -1952,7 +1886,7 @@ bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, if (aligned) { size_t need = (size_t)HQ*T*S*2; if (!g_attn_scores || g_attn_scores.length < need) - g_attn_scores = [ctx().dev newBufferWithLength:need options:MTLResourceStorageModeShared]; + g_attn_scores = owned_shared(need); id sbuf = g_attn_scores; if (sbuf) { float beta = 0.0f; @@ -2507,6 +2441,7 @@ void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, const uint32_t pgb = (gs * 4u + 7u) / 8u; ResW& rw = resident(W); + if (!rw.ok) { cactus_quant_matmul(W, A, M, C); return; } id bx = buf(A, (size_t)K * sizeof(__fp16)); id brecip = rw.recip; id bls = rw.lsign; @@ -2515,31 +2450,30 @@ void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, id bpk = rw.packed; id bcb = rw.codebook; id bnorm = rw.norms; - id bcode = [ctx().dev newBufferWithLength:(size_t)ng * gs * sizeof(__fp16) - options:MTLResourceStorageModeShared]; - id by = [ctx().dev newBufferWithLength:(size_t)N * sizeof(__fp16) - options:MTLResourceStorageModeShared]; + id bcode = owned_shared((size_t)ng * gs * sizeof(__fp16)); + id by = owned_shared((size_t)N * sizeof(__fp16)); id cmd = [ctx().queue commandBuffer]; id e = [cmd computeCommandEncoder]; bool simdT = (gs==128u && ctx().psoTsimd); [e setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; - [e setBuffer:bx offset:0 atIndex:0]; [e setBuffer:brecip offset:0 atIndex:1]; - [e setBuffer:bls offset:0 atIndex:2]; [e setBuffer:brs offset:0 atIndex:3]; - [e setBuffer:bperm offset:0 atIndex:4]; [e setBuffer:bcode offset:0 atIndex:5]; + [e setBuffer:bx offset:0 atIndex:0]; [e setBuffer:brecip offset:rw.rc_off atIndex:1]; + [e setBuffer:bls offset:rw.ls_off atIndex:2]; [e setBuffer:brs offset:rw.rs_off atIndex:3]; + [e setBuffer:bperm offset:rw.pm_off atIndex:4]; [e setBuffer:bcode offset:0 atIndex:5]; [e setBytes:&gs length:sizeof(uint32_t) atIndex:6]; [e setThreadgroupMemoryLength:gs * sizeof(float) atIndex:0]; [e dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; [e setComputePipelineState:ctx().psoG]; - [e setBuffer:bcode offset:0 atIndex:0]; [e setBuffer:bpk offset:0 atIndex:1]; - [e setBuffer:bcb offset:0 atIndex:2]; [e setBuffer:bnorm offset:0 atIndex:3]; + [e setBuffer:bcode offset:0 atIndex:0]; [e setBuffer:bpk offset:rw.packed_off atIndex:1]; + [e setBuffer:bcb offset:rw.cb_off atIndex:2]; [e setBuffer:bnorm offset:rw.norms_off atIndex:3]; [e setBuffer:by offset:0 atIndex:4]; [e setBytes:&gs length:sizeof(uint32_t) atIndex:5]; [e setBytes:&ng length:sizeof(uint32_t) atIndex:6]; [e setBytes:&pgb length:sizeof(uint32_t) atIndex:7]; [e setBytes:&N length:sizeof(uint32_t) atIndex:8]; + [e setBytes:&rw.il length:sizeof(uint32_t) atIndex:9]; const uint32_t ROWS = 8; [e dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; From 0b6feda3ba7fdfc6e08f254ad3be35de6714ea1d Mon Sep 17 00:00:00 2001 From: HenryNdubuaku Date: Tue, 7 Jul 2026 17:56:56 +0100 Subject: [PATCH 36/36] update default model IDs to match existing configurations --- README.md | 18 ++++++------------ python/cactus/cli/common.py | 4 ++-- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1972ee094..9fca47a08 100644 --- a/README.md +++ b/README.md @@ -11,19 +11,13 @@ A hybrid edge-cloud AI engine for mobile devices & wearables. -- **Fast & accurate:** fastest inference on ARM CPU, Cactus quants at 4-bit matches f16 -- **Low RAM:** zero-copy memory mapping ensures 10x lower RAM use than other engines -- **Multimodal:** one engine for speech, vision, and language models -- **Cloud fallback:** automatically route requests to cloud models if needed -- **Model-Agnostic:** Custom PyTorch models can be exported to the Cactus runtime. - ``` ┌─────────────────┐ │ Cactus Engine │ ←── OpenAI-compatible APIs for text, speech, and vision. └─────────────────┘ │ ┌─────────────────┐ -│ Cactus Graph │ ←── Zero-copy computation graph ensures 10x lower RAM +│ Cactus Graph │ ←── Zero-copy computation graph └─────────────────┘ │ ┌─────────────────┐ @@ -128,14 +122,14 @@ graph.hard_reset(); ## Benchmarks -- LLM: Gemma-4-E2B-CQ4 (CPU, no speculative decode), 1k-prefill tps / 100-decode tps -- VLM: Gemma-4-E2B-CQ4 (NPU prefill, CPU decode), 256px input, latency / decode tps -- Transcribe: Parakeet-TDT-0.6B-CQ4 (NPU prefill, CPU decode), 20s audio, latency / decode tps +- LLM: Gemma-4-E2B-CQ4, 1k-prefill tps / 100-decode tps +- VLM: Gemma-4-E2B-CQ4 256px input, latency / decode tps +- Transcribe: Parakeet-TDT-0.6B-CQ4, 20s audio, latency / decode tps - Missing latency == no NPU support for device -| Device | LLM | VLM | Transcribe | RAM | +| Device | LLM | VLM | Transcribe | 1k-Context RAM | |--------|----------|------------|---------------|-----| -| Mac M4 Pro | 324 / 39 | 1.2s / 48 | 0.2s / 10.6M | 1385 MB | +| Mac M4 Pro | 2619tps / 139tps | 0.4s / 160tps | 0.1s / 11Mtps | 1327 MB | | Mac M3 Pro | 390 / 26 | 2.76s / 28.06 | 0.32s / 2.29M | 1376 MB | | iPhone 17 Pro | - | - | - | - | | iPhone 13 Mini | - | - | - | - | diff --git a/python/cactus/cli/common.py b/python/cactus/cli/common.py index 569994792..345dc8e81 100644 --- a/python/cactus/cli/common.py +++ b/python/cactus/cli/common.py @@ -35,8 +35,8 @@ def transpiled_root() -> Path: DEFAULT_MODEL_ID = "google/gemma-4-E2B-it" DEFAULT_TRANSCRIPTION_MODEL_ID = "nvidia/parakeet-tdt-0.6b-v3" -DEFAULT_TEST_MODEL_ID = "LiquidAI/LFM2-VL-450M" -DEFAULT_TEST_TRANSCRIPTION_MODEL_ID = "openai/whisper-base" +DEFAULT_TEST_MODEL_ID = DEFAULT_MODEL_ID +DEFAULT_TEST_TRANSCRIPTION_MODEL_ID = DEFAULT_TRANSCRIPTION_MODEL_ID # Vendor-specific weight-bundle variants; add a new vendor by appending its