diff --git a/README.md b/README.md index 327c8d5bc..9fca47a08 100644 --- a/README.md +++ b/README.md @@ -11,19 +11,13 @@ A hybrid edge-cloud AI engine for mobile devices & wearables. -- **Fast & accurate:** fastest inference on ARM CPU, Cactus quants at 4-bit matches f16 -- **Low RAM:** zero-copy memory mapping ensures 10x lower RAM use than other engines -- **Multimodal:** one engine for speech, vision, and language models -- **Cloud fallback:** automatically route requests to cloud models if needed -- **Model-Agnostic:** Custom PyTorch models can be exported to the Cactus runtime. - ``` ┌─────────────────┐ │ Cactus Engine │ ←── OpenAI-compatible APIs for text, speech, and vision. └─────────────────┘ │ ┌─────────────────┐ -│ Cactus Graph │ ←── Zero-copy computation graph ensures 10x lower RAM +│ Cactus Graph │ ←── Zero-copy computation graph └─────────────────┘ │ ┌─────────────────┐ @@ -128,14 +122,14 @@ graph.hard_reset(); ## Benchmarks -- LLM: Gemma-4-E2B-CQ4 (CPU, no speculative decode), 1k-prefill tps / 100-decode tps -- VLM: Gemma-4-E2B-CQ4 (NPU prefill, CPU decode), 256px input, latency / decode tps -- Transcribe: Parakeet-TDT-0.6B-CQ4 (NPU prefill, CPU decode), 20s audio, latency / decode tps +- LLM: Gemma-4-E2B-CQ4, 1k-prefill tps / 100-decode tps +- VLM: Gemma-4-E2B-CQ4 256px input, latency / decode tps +- Transcribe: Parakeet-TDT-0.6B-CQ4, 20s audio, latency / decode tps - Missing latency == no NPU support for device -| Device | LLM | VLM | Transcribe | RAM | +| Device | LLM | VLM | Transcribe | 1k-Context RAM | |--------|----------|------------|---------------|-----| -| Mac M4 Pro | 324 / 39 | 1.2s / 48 | 0.2s / 10.6M | 1385 MB | +| Mac M4 Pro | 2619tps / 139tps | 0.4s / 160tps | 0.1s / 11Mtps | 1327 MB | | Mac M3 Pro | 390 / 26 | 2.76s / 28.06 | 0.32s / 2.29M | 1376 MB | | iPhone 17 Pro | - | - | - | - | | iPhone 13 Mini | - | - | - | - | @@ -196,7 +190,8 @@ graph.hard_reset(); │ │ │ cactus run [model|path] run a model (downloads if needed) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ +│ --backend auto|cpu|metal inference backend (default: auto) │ │ --image image file for VLM inference │ │ --audio audio file for audio chat │ │ --system system prompt │ @@ -209,20 +204,20 @@ graph.hard_reset(); │ --file audio file to transcribe (WAV) │ │ --language language code (default: en) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ │ │ cactus download [model] get a bundle (prebuilt, else build) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ │ │ cactus convert [dir] HuggingFace -> runnable cactus bundle │ │ (CQ weights + runtime graph) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ --lora merge a LoRA adapter before converting │ @@ -233,7 +228,8 @@ graph.hard_reset(); │ --host bind address (default: 127.0.0.1) │ │ --port port (default: 8080) │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ +│ --backend auto|cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ --no-cloud-handoff disable automatic cloud handoff │ @@ -243,7 +239,8 @@ graph.hard_reset(); │ cactus code run the AI coding agent (TUI / print) │ │ --serve-model auto-start a server with this model │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ +│ --backend auto|cpu|metal inference backend (default: auto) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild from source │ │ --host server address (default: 127.0.0.1) │ @@ -267,7 +264,7 @@ graph.hard_reset(); │ --model default: LiquidAI/LFM2-VL-450M │ │ --transcription-model default: openai/whisper-base │ │ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --platform auto|cpu|apple target platform (default: auto) │ +│ --weights general|apple weights bundle variant (default: general) │ │ --token HuggingFace token (gated models) │ │ --reconvert force local rebuild of test models │ │ --suite run a single test suite by name │ diff --git a/apple/CMakeLists.txt b/apple/CMakeLists.txt index 1afd6078f..cdd95297d 100644 --- a/apple/CMakeLists.txt +++ b/apple/CMakeLists.txt @@ -1,8 +1,8 @@ cmake_minimum_required(VERSION 3.10) project(CactusApple LANGUAGES CXX) -set(CMAKE_OSX_DEPLOYMENT_TARGET "13.0" CACHE STRING "") -set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "13.0" CACHE STRING "") +set(CMAKE_OSX_DEPLOYMENT_TARGET "13.3" CACHE STRING "") +set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "16.4" CACHE STRING "") set(CMAKE_BUILD_TYPE Release CACHE STRING "") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) @@ -27,7 +27,7 @@ if(BUILD_SHARED_LIBS) ) target_link_libraries(cactus_framework PUBLIC - "-framework Accelerate" "-framework CoreML" "-framework Foundation" + "-framework Accelerate" "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders" "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" ) diff --git a/apple/Info.plist b/apple/Info.plist index 91f8988f6..401ef4214 100644 --- a/apple/Info.plist +++ b/apple/Info.plist @@ -19,6 +19,6 @@ CFBundleVersion 1.0.0 MinimumOSVersion - 13.0 + 13.3 diff --git a/apple/build.sh b/apple/build.sh index 69b2ebe75..f71ee5c5a 100755 --- a/apple/build.sh +++ b/apple/build.sh @@ -1,4 +1,6 @@ #!/bin/bash +IOS_MIN_VERSION=16.4 +MACOS_MIN_VERSION=13.3 set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -59,7 +61,7 @@ function build_static_library() { cmake -DCMAKE_SYSTEM_NAME=iOS \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=$IOS_MIN_VERSION \ -DCMAKE_OSX_SYSROOT="$IOS_SDK_PATH" \ -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \ -DBUILD_SHARED_LIBS=OFF \ @@ -86,7 +88,7 @@ function build_static_library() { cmake -DCMAKE_SYSTEM_NAME=iOS \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=$IOS_MIN_VERSION \ -DCMAKE_OSX_SYSROOT="$IOS_SIM_SDK_PATH" \ -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \ -DBUILD_SHARED_LIBS=OFF \ @@ -110,7 +112,7 @@ function build_framework() { -DCMAKE_SYSTEM_NAME=$1 \ -DCMAKE_OSX_ARCHITECTURES="$2" \ -DCMAKE_OSX_SYSROOT=$3 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=$([ "$1" = "iOS" ] && echo "$IOS_MIN_VERSION" || echo "$MACOS_MIN_VERSION") \ -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \ -DBUILD_SHARED_LIBS=ON \ -DCACTUS_CURL_ROOT="$CACTUS_CURL_ROOT" \ diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index 8183d9514..7e057285e 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -56,7 +56,7 @@ target_link_libraries(cactus_engine PUBLIC cactus_graph) if(APPLE) target_link_libraries(cactus_engine PUBLIC - "-framework CoreML" "-framework Foundation" + "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders" "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" ) @@ -156,7 +156,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) target_link_libraries(cactus_engine_shared PRIVATE "${_curl}") endif() target_link_libraries(cactus_engine_shared PRIVATE - "-framework Accelerate" "-framework CoreML" "-framework Foundation" + "-framework Accelerate" "-framework CoreML" "-framework Foundation" "-framework Metal" "-framework MetalPerformanceShaders" "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" ) else() diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index 376628563..5dacb0bf6 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -34,6 +34,8 @@ CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); +CACTUS_FFI_EXPORT int cactus_set_backend(const char* backend); + CACTUS_FFI_EXPORT int cactus_complete( cactus_model_t model, const char* messages_json, diff --git a/cactus-engine/src/complete.cpp b/cactus-engine/src/complete.cpp index 85273c3c9..0821cf9bb 100644 --- a/cactus-engine/src/complete.cpp +++ b/cactus-engine/src/complete.cpp @@ -4,6 +4,7 @@ #include "chat_tools.h" #include "telemetry.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include "wav.h" #include #include @@ -796,6 +797,10 @@ int cactus_complete( const uint8_t* pcm_buffer, size_t pcm_buffer_size ) { + struct MetalTrimGuard { + ~MetalTrimGuard() { cactus_metal_trim_prefill_cache(); } + } metal_trim_guard; + if (!model) { std::string error_msg = last_error_message.empty() ? "Model not initialized. Check model path and files." : last_error_message; @@ -824,6 +829,9 @@ int cactus_complete( } auto* tokenizer = handle->model->get_tokenizer(); auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, true, true, pcm_buffer, pcm_buffer_size); + if (prompt.options.sample_seed != 0) { + handle->model->set_sample_seed(prompt.options.sample_seed); + } CACTUS_LOG_DEBUG("complete", "Prompt tokens: " << prompt.tokens.size() << ", max_tokens: " << prompt.options.max_tokens); diff --git a/cactus-engine/src/constraints.cpp b/cactus-engine/src/constraints.cpp index ec6731dbd..c966b7000 100644 --- a/cactus-engine/src/constraints.cpp +++ b/cactus-engine/src/constraints.cpp @@ -124,11 +124,11 @@ void ToolCallConstrainer::mark_trie_bias(const TrieNode* node, char terminator, } if (!has_valid) return; + dense_bias_.assign(valid.size(), TRIE_BLOCK_BIAS); for (size_t token_id = 0; token_id < valid.size(); ++token_id) { - if (!valid[token_id]) { - current_bias_[static_cast(token_id)] = TRIE_BLOCK_BIAS; - } + if (valid[token_id]) dense_bias_[token_id] = 0.0f; } + dense_ready_ = true; } void ToolCallConstrainer::reset_constraint_state() { @@ -432,6 +432,7 @@ void ToolCallConstrainer::init(Config::ModelType model_type, tokenizer_ = tokenizer; generated_text_.clear(); current_bias_.clear(); + dense_ready_ = false; forced_tag_progress_ = 0; const bool model_supported = is_needle() || Config::is_gemma_family(model_type_); active_ = model_supported && !tool_specs_.empty() && tokenizer != nullptr; @@ -520,6 +521,7 @@ void ToolCallConstrainer::update(uint32_t token_id, const std::string& decoded_t void ToolCallConstrainer::compute_bias() { current_bias_.clear(); + dense_ready_ = false; if (!active_) return; @@ -615,6 +617,7 @@ void ToolCallConstrainer::compute_bias() { void ToolCallConstrainer::reset() { generated_text_.clear(); current_bias_.clear(); + dense_ready_ = false; forced_tag_progress_ = 0; reset_constraint_state(); diff --git a/cactus-engine/src/embed.cpp b/cactus-engine/src/embed.cpp index 6a380836a..33e9daf7f 100644 --- a/cactus-engine/src/embed.cpp +++ b/cactus-engine/src/embed.cpp @@ -1,6 +1,7 @@ #include "../cactus_engine.h" #include "utils.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include "wav.h" #include #include @@ -37,6 +38,9 @@ int cactus_embed( size_t* embedding_dim, bool normalize ) { + struct MetalTrimGuard { + ~MetalTrimGuard() { cactus_metal_trim_prefill_cache(); } + } metal_trim_guard; if (!model || !text || !embeddings_buffer || buffer_size == 0) { CACTUS_LOG_ERROR("embed", "Invalid parameters for text embedding"); return -1; @@ -90,6 +94,9 @@ int cactus_image_embed( size_t buffer_size, size_t* embedding_dim ) { + struct MetalTrimGuard { + ~MetalTrimGuard() { cactus_metal_trim_prefill_cache(); } + } metal_trim_guard; if (!model || !image_path || !embeddings_buffer || buffer_size == 0) { CACTUS_LOG_ERROR("image_embed", "Invalid parameters for image embedding"); return -1; @@ -130,6 +137,9 @@ int cactus_audio_embed( size_t buffer_size, size_t* embedding_dim ) { + struct MetalTrimGuard { + ~MetalTrimGuard() { cactus_metal_trim_prefill_cache(); } + } metal_trim_guard; if (!model || !audio_path || !embeddings_buffer || buffer_size == 0) { CACTUS_LOG_ERROR("audio_embed", "Invalid parameters for audio embedding"); return -1; diff --git a/cactus-engine/src/engine.h b/cactus-engine/src/engine.h index 46bcdf415..d0cffb294 100644 --- a/cactus-engine/src/engine.h +++ b/cactus-engine/src/engine.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -522,6 +523,7 @@ class ToolCallConstrainer { Tokenizer* tokenizer); const std::unordered_map& get_bias() const { return current_bias_; } + const std::vector* get_dense_bias() const { return dense_ready_ ? &dense_bias_ : nullptr; } void update(uint32_t token_id, const std::string& decoded_text); @@ -561,6 +563,8 @@ class ToolCallConstrainer { std::unordered_set close_brace_tokens_; std::unordered_map current_bias_; + std::vector dense_bias_; + bool dense_ready_ = false; void compute_bias(); void tokenize_grammar_elements(); @@ -634,6 +638,7 @@ class Model { uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, float min_p = 0.15f, float repetition_penalty = 1.1f); + void set_sample_seed(uint64_t seed) { sample_rng_.seed(static_cast(seed)); } bool prefill_and_sample_first_token(const std::vector& tokens, uint32_t& out_token, float* out_uncertainty = nullptr); @@ -807,7 +812,7 @@ class Model { bool load_component_graph(Component& comp); void unload_component_graph(Component& comp); bool bind_runtime_buffers(Component& comp); - void run_step(uint32_t token_id, size_t position, bool read_logits); + void run_step(uint32_t token_id, size_t position, bool read_logits, bool use_fused = true); void run_step_batch(const std::vector& token_ids, const std::vector& positions); void set_component_batch(Component& comp, size_t batch); size_t decoder_cache_num_slots(); @@ -845,6 +850,9 @@ class Model { size_t chunk_tokens, const std::vector& tokens, size_t processed, size_t start_position); void run_full_context_text(); + void prepare_sampling_context(float repetition_penalty); + uint32_t sample_component_logits(Component& comp, float temperature, float top_p, size_t top_k, + float min_p, bool greedy, float* out_uncertainty); uint32_t argmax_component_logits(Component& comp, size_t logit_row = std::numeric_limits::max(), float* out_uncertainty = nullptr); uint32_t argmax_logits_at(const BufferDesc& desc, void* ptr, size_t row_off, float* out_uncertainty); @@ -942,6 +950,14 @@ class Model { static constexpr size_t MAX_TOKEN_HISTORY = 128; std::vector token_history_; + std::vector samp_recent_; + std::vector samp_bias_dense_; + bool samp_has_bias_ = false; + float samp_penalty_ = 1.0f; + bool samp_ctx_active_ = false; + std::mt19937 sample_rng_{std::random_device{}()}; + FusedEmbedCtx fused_embed_ctx_; + int ple_probe_state_ = 0; ToolCallConstrainer tool_constrainer_; std::unordered_map vocab_bias_; diff --git a/cactus-engine/src/init.cpp b/cactus-engine/src/init.cpp index 7672d88f2..ca11df7fa 100644 --- a/cactus-engine/src/init.cpp +++ b/cactus-engine/src/init.cpp @@ -349,6 +349,8 @@ bool matches_stop_sequence(const std::vector& generated_tokens, extern "C" { +int cactus_set_backend(const char* backend) { return cactus_backend_select(backend); } + cactus_model_t cactus_init(const char* model_path, const char* corpus_dir, bool cache_index) { constexpr size_t DEFAULT_CONTEXT_SIZE = 512; // matches default sliding window size diff --git a/cactus-engine/src/model.cpp b/cactus-engine/src/model.cpp index 412ccc2b2..c8a5cb6c9 100644 --- a/cactus-engine/src/model.cpp +++ b/cactus-engine/src/model.cpp @@ -562,6 +562,8 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, } else if (decode_route_ != DecodeRoute::ENCODER_CROSS_KV_STEP && !components_.count("audio_encoder")) { CACTUS_LOG_WARN("model", "Bundle has no decoder_prefill_chunk component; prompts will prefill token-by-token (prefill speed ~= decode speed)."); } + if (decoder_ && decoder_->graph) decoder_->graph->prewarm_metal_quant_weights(); + if (decoder_prefill_ && decoder_prefill_->graph) decoder_prefill_->graph->prewarm_metal_quant_weights(); if (components_.count("decoder_embed_chunk") && components_.at("decoder_embed_chunk").graph) { decoder_embed_ = &components_.at("decoder_embed_chunk"); } @@ -600,6 +602,21 @@ bool Model::init(const std::string& bundle_dir, size_t context_size, bound.insert(comp); } + if (decoder_prefill_ && decoder_prefill_->graph) { + try { + if (prefill_encoder_ && prefill_encoder_->graph) { + for (auto& b : prefill_encoder_->input_buffers) std::fill(b.begin(), b.end(), 0); + prefill_encoder_->graph->execute(); + reset_component_cache_states(*prefill_encoder_); + } + for (auto& b : decoder_prefill_->input_buffers) std::fill(b.begin(), b.end(), 0); + decoder_prefill_->graph->execute(); + reset_component_cache_states(*decoder_prefill_); + } catch (const std::exception& e) { + CACTUS_LOG_WARN("model", std::string("prefill warmup skipped: ") + e.what()); + } + } + if (vision_encoder_ && tokenizer_ && !vision_encoder_->output_node_ids.empty()) { size_t out_node = static_cast(vision_encoder_->output_node_ids[0]); const auto& desc = vision_encoder_->graph->get_output_buffer(out_node); @@ -790,6 +807,7 @@ bool Model::load_component_graph(Component& comp) { } void Model::unload_component_graph(Component& comp) { + if (cactus_backend_metal()) return; if (comp.graph) { comp.graph->release_runtime_buffers(); comp.graph->release_all_weight_pages(); @@ -877,17 +895,36 @@ void Model::copy_encoder_outputs_to_decoder(const Component& enc) { } } -void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/) { +void Model::run_step(uint32_t token_id, size_t position, bool /*read_logits*/, bool use_fused) { if (decode_route_ == DecodeRoute::DIRECT_DECODER_STEP) { write_int_input(*decoder_, "input_ids", static_cast(token_id)); write_int_input(*decoder_, "position_ids", static_cast(position)); + if (!use_fused || !cactus_backend_metal()) cactus_graph_set_prefill_consistent(true); decoder_->graph->execute(); + cactus_graph_set_prefill_consistent(false); maybe_capture_handoff_probe_hidden(*decoder_); return; } + if (!use_fused) cactus_graph_set_prefill_consistent(true); + if (use_fused && cactus_backend_metal()) { + if (ple_probe_state_ == 0) + ple_probe_state_ = encoder_->graph->extract_ple_pathway(fused_embed_ctx_) ? 1 : 2; + if (ple_probe_state_ == 1) { + fused_embed_ctx_.token_id = static_cast(token_id); + fused_embed_ctx_.position = static_cast(position); + cactus_graph_set_fused_embed(&fused_embed_ctx_); + write_int_input(*decoder_, "position_ids", static_cast(position)); + decoder_->graph->execute(); + cactus_graph_set_fused_embed(nullptr); + cactus_graph_set_prefill_consistent(false); + maybe_capture_handoff_probe_hidden(*decoder_); + return; + } + } run_encoder_step(token_id, position); copy_component_outputs_to_inputs(*encoder_, *decoder_); decoder_->graph->execute(); + cactus_graph_set_prefill_consistent(false); maybe_capture_handoff_probe_hidden(*decoder_); } @@ -983,9 +1020,13 @@ std::vector> Model::generate_batch(const std::vector 1 || cactus_backend_metal()) { + exec_batch = std::max(batch, decoder_cache_num_slots()); + } reset_component_cache_states(*decoder_); - if (cached) set_component_batch(*encoder_, batch); - set_component_batch(*decoder_, batch); + if (cached) set_component_batch(*encoder_, exec_batch); + set_component_batch(*decoder_, exec_batch); std::vector> out(batch); std::vector fed(batch, 0); @@ -999,13 +1040,17 @@ std::vector> Model::generate_batch(const std::vector tokens(batch); - std::vector positions(batch); + std::vector tokens(exec_batch); + std::vector positions(exec_batch); while (remaining > 0) { for (size_t b = 0; b < batch; ++b) { positions[b] = fed[b]; tokens[b] = (fed[b] < prompts[b].size()) ? prompts[b][fed[b]] : last[b]; } + for (size_t b = batch; b < exec_batch; ++b) { + positions[b] = positions[0]; + tokens[b] = tokens[0]; + } run_step_batch(tokens, positions); std::vector sampled = argmax_component_logits_batch(*decoder_, batch); for (size_t b = 0; b < batch; ++b) { @@ -1579,8 +1624,10 @@ void Model::run_vision_encoder(const std::string& image_path) { media_feature_shapes_[name] = desc.shape; media_feature_precisions_[name] = desc.precision; } - vision_encoder_->graph->release_runtime_buffers(); - vision_encoder_->graph->release_all_weight_pages(); + if (!cactus_backend_metal()) { + vision_encoder_->graph->release_runtime_buffers(); + vision_encoder_->graph->release_all_weight_pages(); + } unload_component_graph(*vision_encoder_); } @@ -1794,8 +1841,10 @@ void Model::run_audio_encoder_messages(const std::vector>& au if (mel.empty()) continue; run_audio_encoder(mel); } - audio_encoder_->graph->release_runtime_buffers(); - audio_encoder_->graph->release_all_weight_pages(); + if (!cactus_backend_metal()) { + audio_encoder_->graph->release_runtime_buffers(); + audio_encoder_->graph->release_all_weight_pages(); + } unload_component_graph(*audio_encoder_); } @@ -1899,13 +1948,32 @@ void Model::run_audio_encoder(const std::vector& audio_features) { } } +static float uncertainty_from_margin(float best, float second) { + float confidence = 1.0f; + if (std::isfinite(best) && std::isfinite(second)) { + float margin = std::max(-60.0f, std::min(60.0f, best - second)); + confidence = 1.0f / (1.0f + std::exp(-margin)); + } + return std::max(0.0f, std::min(1.0f, 1.0f - confidence)); +} + uint32_t Model::argmax_logits_at(const BufferDesc& desc, void* ptr, size_t row_off, float* out_uncertainty) { size_t vocab = desc.shape.empty() ? 0 : desc.shape.back(); uint32_t best = 0; float best_v = -std::numeric_limits::infinity(); float second_v = -std::numeric_limits::infinity(); const auto& tool_bias = tool_constrainer_.get_bias(); + const std::vector* tool_dense = tool_constrainer_.get_dense_bias(); + { + uint32_t gidx; float gbest, gsecond; + if (!tool_dense && tool_bias.empty() && vocab_bias_.empty() && suppressed_token_id_ < 0 && + cactus_graph_metal_argmax(&gidx, &gbest, &gsecond)) { + if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(gbest, gsecond); + return gidx; + } + } auto score_with_bias = [&](size_t token_id, float value) { + if (tool_dense && token_id < tool_dense->size()) value += (*tool_dense)[token_id]; auto tool_it = tool_bias.find(static_cast(token_id)); if (tool_it != tool_bias.end()) value += tool_it->second; auto vocab_it = vocab_bias_.find(static_cast(token_id)); @@ -1933,14 +2001,7 @@ uint32_t Model::argmax_logits_at(const BufferDesc& desc, void* ptr, size_t row_o int8_t* p = static_cast(ptr) + row_off; for (size_t i = 0; i < vocab; ++i) observe_logit(i, static_cast(p[i])); } - if (out_uncertainty) { - float confidence = 1.0f; - if (std::isfinite(best_v) && std::isfinite(second_v)) { - float margin = std::max(-60.0f, std::min(60.0f, best_v - second_v)); - confidence = 1.0f / (1.0f + std::exp(-margin)); - } - *out_uncertainty = std::max(0.0f, std::min(1.0f, 1.0f - confidence)); - } + if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(best_v, second_v); return best; } @@ -1980,6 +2041,162 @@ uint32_t Model::argmax_last_logits(float* out_uncertainty) { return argmax_component_logits(*decoder_, std::numeric_limits::max(), out_uncertainty); } +void Model::prepare_sampling_context(float repetition_penalty) { + samp_recent_.clear(); + samp_has_bias_ = false; + samp_penalty_ = repetition_penalty; + constexpr size_t kPenaltyWindow = 64; + size_t start = token_history_.size() > kPenaltyWindow ? token_history_.size() - kPenaltyWindow : 0; + for (size_t i = start; i < token_history_.size(); ++i) { + uint32_t id = token_history_[i]; + bool dup = false; + for (uint32_t r : samp_recent_) if (r == id) { dup = true; break; } + if (!dup) samp_recent_.push_back(id); + } + const std::vector* tool_dense = tool_constrainer_.get_dense_bias(); + const auto& tool_bias = tool_constrainer_.get_bias(); + if (tool_dense || !tool_bias.empty() || !vocab_bias_.empty()) { + size_t n = tokenizer_ ? tokenizer_->get_vocab_size() : 0; + if (tool_dense && tool_dense->size() > n) n = tool_dense->size(); + if (n > 0) { + if (tool_dense) samp_bias_dense_.assign(tool_dense->begin(), tool_dense->end()); + else samp_bias_dense_.assign(n, 0.0f); + if (samp_bias_dense_.size() < n) samp_bias_dense_.resize(n, 0.0f); + for (const auto& kv : tool_bias) + if (kv.first < samp_bias_dense_.size()) samp_bias_dense_[kv.first] += kv.second; + for (const auto& kv : vocab_bias_) + if (kv.first < samp_bias_dense_.size()) samp_bias_dense_[kv.first] += kv.second; + samp_has_bias_ = true; + } + } + samp_ctx_active_ = (repetition_penalty != 1.0f && !samp_recent_.empty()) || + samp_has_bias_ || suppressed_token_id_ >= 0; + if (samp_ctx_active_) { + cactus_graph_set_sampling(samp_recent_.data(), (int)samp_recent_.size(), repetition_penalty, + samp_has_bias_ ? samp_bias_dense_.data() : nullptr, + samp_has_bias_ ? samp_bias_dense_.size() : 0, + suppressed_token_id_); + } else { + cactus_graph_clear_sampling(); + } +} + +uint32_t Model::sample_component_logits(Component& comp, float temperature, float top_p, size_t top_k, + float min_p, bool greedy, float* out_uncertainty) { + size_t out_node = static_cast(comp.output_node_ids.empty() ? 0 : comp.output_node_ids[0]); + const auto& desc = comp.graph->get_output_buffer(out_node); + void* ptr = comp.graph->get_output(out_node); + size_t vocab = desc.shape.empty() ? 0 : desc.shape.back(); + size_t seq = desc.shape.size() >= 2 ? desc.shape[desc.shape.size() - 2] : 1; + size_t row = seq > 0 ? seq - 1 : 0; + if (decode_route_ == DecodeRoute::FULL_CONTEXT_TEXT) { + row = std::min(last_logit_position_, seq > 0 ? seq - 1 : 0); + } + size_t off = row * vocab; + if (vocab == 0 || (desc.precision != Precision::FP16 && desc.precision != Precision::FP32)) { + return argmax_component_logits(comp, std::numeric_limits::max(), out_uncertainty); + } + const bool fp16 = desc.precision == Precision::FP16; + __fp16* h = fp16 ? static_cast<__fp16*>(ptr) + off : nullptr; + float* f = fp16 ? nullptr : static_cast(ptr) + off; + auto get = [&](size_t i) -> float { return fp16 ? (float)h[i] : f[i]; }; + auto put = [&](size_t i, float v) { if (fp16) h[i] = (__fp16)v; else f[i] = v; }; + + if (samp_ctx_active_ && !cactus_graph_metal_adjusted()) { + if (samp_penalty_ != 1.0f) { + for (uint32_t id : samp_recent_) { + if (id >= vocab) continue; + float v = get(id); + put(id, v > 0.0f ? v / samp_penalty_ : v * samp_penalty_); + } + } + if (suppressed_token_id_ >= 0 && (size_t)suppressed_token_id_ < vocab) + put((size_t)suppressed_token_id_, -65504.0f); + } + if (greedy) { + uint32_t gidx; float gbest, gsecond; + if ((!samp_has_bias_ || cactus_graph_metal_argmax_biased()) && + cactus_graph_metal_argmax(&gidx, &gbest, &gsecond)) { + if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(gbest, gsecond); + return gidx; + } + } + const float* bd = samp_has_bias_ ? samp_bias_dense_.data() : nullptr; + const size_t bn = samp_has_bias_ ? samp_bias_dense_.size() : 0; + auto biased = [&](size_t i) -> float { + float v = get(i); + if (i < bn) v += bd[i]; + return v; + }; + + if (greedy) { + uint32_t best = 0; + float bv = -std::numeric_limits::infinity(), sv = bv; + for (size_t i = 0; i < vocab; ++i) { + float v = biased(i); + if (v > bv) { sv = bv; bv = v; best = (uint32_t)i; } + else if (v > sv) sv = v; + } + if (out_uncertainty) *out_uncertainty = uncertainty_from_margin(bv, sv); + return best; + } + + size_t K = std::min(std::max(top_k, 1), 512); + std::vector> cand; + cand.reserve(2 * K + 16); + float kmin = -std::numeric_limits::infinity(); + for (size_t i = 0; i < vocab; ++i) { + float v = biased(i); + if (v <= kmin) continue; + cand.emplace_back(v, (uint32_t)i); + if (cand.size() >= 2 * K) { + std::nth_element(cand.begin(), cand.begin() + (K - 1), cand.end(), + [](const auto& a, const auto& b) { return a.first > b.first; }); + cand.resize(K); + kmin = cand.back().first; + for (const auto& c : cand) if (c.first < kmin) kmin = c.first; + } + } + std::sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); + if (cand.size() > K) cand.resize(K); + if (cand.empty()) return 0; + if (out_uncertainty) { + float sv = cand.size() > 1 ? cand[1].first : -std::numeric_limits::infinity(); + *out_uncertainty = uncertainty_from_margin(cand[0].first, sv); + } + if (cand.size() == 1) return cand[0].second; + + float t = std::max(temperature, 1e-3f); + float maxl = cand[0].first; + std::vector probs(cand.size()); + double denom = 0.0; + for (size_t i = 0; i < cand.size(); ++i) { + probs[i] = std::exp((cand[i].first - maxl) / t); + denom += probs[i]; + } + size_t keep = cand.size(); + if (top_p > 0.0f && top_p < 1.0f) { + double cum = 0.0; + for (size_t i = 0; i < cand.size(); ++i) { + cum += probs[i] / denom; + if (cum >= top_p) { keep = i + 1; break; } + } + } + if (min_p > 0.0f) { + float cut = min_p * probs[0]; + while (keep > 1 && probs[keep - 1] < cut) --keep; + } + double mass = 0.0; + for (size_t i = 0; i < keep; ++i) mass += probs[i]; + double r = std::uniform_real_distribution(0.0, mass)(sample_rng_); + double cum = 0.0; + for (size_t i = 0; i < keep; ++i) { + cum += probs[i]; + if (r <= cum) return cand[i].second; + } + return cand[keep - 1].second; +} + bool Model::prefill_and_sample_first_token(const std::vector& tokens, uint32_t& out_token, float* out_uncertainty) { reset_prefill_stats(); @@ -2034,7 +2251,7 @@ bool Model::prefill_and_sample_first_token(const std::vector& tokens, // Cache already moved into the step component; drop the padded last row and re-run it for logits. set_cache_current_len(*decoder_, tokens.size() - 1); cache_total_seq_len_ = tokens.size() - 1; - run_step(tokens.back(), cache_total_seq_len_, true); + run_step(tokens.back(), cache_total_seq_len_, true, /*use_fused=*/false); ++cache_total_seq_len_; out_token = argmax_last_logits(out_uncertainty); record_sampled_token(out_token); @@ -2045,7 +2262,7 @@ bool Model::prefill_and_sample_first_token(const std::vector& tokens, cache_total_seq_len_ += chunked.logical_tokens; } for (size_t i = chunked.logical_tokens; i < tokens.size(); ++i) { - run_step(tokens[i], cache_total_seq_len_, i + 1 == tokens.size()); + run_step(tokens[i], cache_total_seq_len_, i + 1 == tokens.size(), /*use_fused=*/false); ++cache_total_seq_len_; } last_prefill_scalar_tail_tokens_ = tokens.size() - chunked.logical_tokens; @@ -2076,7 +2293,7 @@ void Model::prefill(const std::vector& tokens, size_t /*chunk_size*/, ChunkedPrefillResult chunked = run_chunked_prefill(tokens, cache_total_seq_len_, get_prefill_chunk_size(), prepare_decode); cache_total_seq_len_ += chunked.logical_tokens; for (size_t i = chunked.logical_tokens; i < tokens.size(); ++i) { - run_step(tokens[i], cache_total_seq_len_, /*read_logits=*/false); + run_step(tokens[i], cache_total_seq_len_, /*read_logits=*/false, /*use_fused=*/false); ++cache_total_seq_len_; } cache_token_ids_.insert(cache_token_ids_.end(), tokens.begin(), tokens.end()); @@ -2987,10 +3204,19 @@ void Model::prefill_with_media(const std::vector& tokens, (void)profile_file; } -uint32_t Model::decode(const std::vector& tokens, float /*temperature*/, float /*top_p*/, - size_t /*top_k*/, const std::string& /*profile_file*/, float* out_entropy, - float /*min_p*/, float /*repetition_penalty*/) { +uint32_t Model::decode(const std::vector& tokens, float temperature, float top_p, + size_t top_k, const std::string& /*profile_file*/, float* out_entropy, + float min_p, float repetition_penalty) { if (tokens.empty()) return 0; + float temp = temperature < 0.0f ? config_.default_temperature : temperature; + float tp = (top_p <= 0.0f || top_p > 1.0f) ? config_.default_top_p : top_p; + size_t tk = top_k == 0 ? config_.default_top_k : top_k; + const bool greedy = temp <= 0.011f; + prepare_sampling_context(repetition_penalty); + struct SampClearGuard { + Model* m; + ~SampClearGuard() { cactus_graph_clear_sampling(); m->samp_ctx_active_ = false; } + } samp_guard{this}; if (decode_route_ == DecodeRoute::ENCODER_CROSS_KV_STEP) { if (!encoder_cross_kv_ready_ && encoder_cross_kv_source_kind_ == "text_tokens") { std::vector source_tokens = tokens; @@ -3022,7 +3248,9 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* run_full_context_text(); cache_total_seq_len_ = context_tokens_.size(); cache_token_ids_ = context_tokens_; - uint32_t result = argmax_last_logits(out_entropy); + uint32_t result = (greedy && !samp_ctx_active_) + ? argmax_last_logits(out_entropy) + : sample_component_logits(*decoder_, temp, tp, tk, min_p, greedy, out_entropy); record_sampled_token(result); return result; } @@ -3033,7 +3261,9 @@ uint32_t Model::decode(const std::vector& tokens, float /*temperature* cache_total_seq_len_ += tokens.size(); cache_token_ids_.insert(cache_token_ids_.end(), tokens.begin(), tokens.end()); maybe_roll_compact(); - uint32_t result = argmax_last_logits(out_entropy); + uint32_t result = (greedy && !samp_ctx_active_) + ? argmax_last_logits(out_entropy) + : sample_component_logits(*decoder_, temp, tp, tk, min_p, greedy, out_entropy); record_sampled_token(result); return result; } @@ -3470,10 +3700,15 @@ std::vector Model::get_image_embeddings(const std::string& image_path) { if (vision_encoder_->logical_outputs.empty()) { throw std::runtime_error("vision_encoder has no logical outputs"); } - const std::string output_name = vision_encoder_->logical_outputs[0]; + std::string output_name = vision_encoder_->logical_outputs[0]; run_vision_encoder(image_path); + if (!media_features_.count(output_name)) { + for (const char* name : {"image_features", "image_embeddings", "vision_features"}) { + if (media_features_.count(name)) { output_name = name; break; } + } + } auto bytes_it = media_features_.find(output_name); auto shape_it = media_feature_shapes_.find(output_name); auto prec_it = media_feature_precisions_.find(output_name); @@ -3490,6 +3725,9 @@ std::vector Model::get_image_embeddings(const std::string& image_path) { media_feature_shapes_.erase(name); media_feature_precisions_.erase(name); } + media_features_.erase(output_name); + media_feature_shapes_.erase(output_name); + media_feature_precisions_.erase(output_name); // run_vision_encoder unloads the graph; restore so subsequent paths that // assume the encoder is loaded (e.g. transcribe_*) keep working. load_component_graph(*vision_encoder_); diff --git a/cactus-engine/src/npu_ane.h b/cactus-engine/src/npu_ane.h index 7594960bf..07b678687 100644 --- a/cactus-engine/src/npu_ane.h +++ b/cactus-engine/src/npu_ane.h @@ -14,7 +14,6 @@ namespace npu { #if CACTUS_HAS_ANE - class ANEEncoder : public NPUEncoder { public: ANEEncoder(); diff --git a/cactus-engine/src/npu_ane.mm b/cactus-engine/src/npu_ane.mm index 99638e2e0..3bcf2b51c 100644 --- a/cactus-engine/src/npu_ane.mm +++ b/cactus-engine/src/npu_ane.mm @@ -1016,7 +1016,7 @@ - (size_t)predictMultiInputWithDict:(NSMutableDictionary create_encoder() { - return std::make_unique(); + return std::make_unique(); } bool is_npu_available() { diff --git a/cactus-engine/src/transcribe.cpp b/cactus-engine/src/transcribe.cpp index cafd98c67..9f2ebe928 100644 --- a/cactus-engine/src/transcribe.cpp +++ b/cactus-engine/src/transcribe.cpp @@ -2,6 +2,7 @@ #include "utils.h" #include "cloud.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include "wav.h" #include #include @@ -163,6 +164,9 @@ int cactus_transcribe( const uint8_t* pcm_buffer, size_t pcm_buffer_size ) { + struct MetalTrimGuard { + ~MetalTrimGuard() { cactus_metal_trim_prefill_cache(); } + } metal_trim_guard; if (validate_audio_params("transcribe", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) { return -1; diff --git a/cactus-engine/src/utils.h b/cactus-engine/src/utils.h index 23f6b399c..afc720a1b 100644 --- a/cactus-engine/src/utils.h +++ b/cactus-engine/src/utils.h @@ -508,6 +508,7 @@ struct InferenceOptions { float top_p = 0.0f; float min_p = 0.15f; float repetition_penalty = 1.1f; + uint64_t sample_seed = 0; float confidence_threshold = -1.0f; size_t top_k = 0; size_t max_tokens = 100; @@ -1619,6 +1620,11 @@ inline InferenceOptions parse_inference_options_json(const std::string& json) { } } + size_t parsed_seed = 0; + if (try_parse_json_uint(json, "seed", parsed_seed)) { + options.sample_seed = parsed_seed; + } + size_t parsed_top_k = options.top_k; if (try_parse_json_uint(json, "top_k", parsed_top_k)) { options.top_k = parsed_top_k; diff --git a/cactus-engine/test.sh b/cactus-engine/test.sh index 25cfc92ec..7d581662c 100755 --- a/cactus-engine/test.sh +++ b/cactus-engine/test.sh @@ -8,6 +8,7 @@ ASSETS_DIR="$(pwd)/tests/assets" IOS_MODE=false ANDROID_MODE=false SUITE="" +BACKEND="auto" while [[ $# -gt 0 ]]; do case $1 in @@ -16,6 +17,7 @@ while [[ $# -gt 0 ]]; do --suite) SUITE="${2:?--suite needs an argument}"; shift 2 ;; --model) CACTUS_TEST_MODEL="${2:?--model needs an argument}"; shift 2 ;; --transcription-model) CACTUS_TEST_TRANSCRIPTION_MODEL="${2:?--transcription-model needs an argument}"; shift 2 ;; + --backend) BACKEND="${2:?--backend needs an argument}"; shift 2 ;; *) echo "Unknown arg: $1" >&2; exit 2 ;; esac done @@ -33,11 +35,11 @@ BUNDLE_DIR="$(require_bundle "$CACTUS_TEST_MODEL" "model")" TRANSCRIPTION_BUNDLE_DIR="$(require_bundle "$CACTUS_TEST_TRANSCRIPTION_MODEL" "transcription-model")" if [ "$IOS_MODE" = true ]; then - export CACTUS_TEST_MODEL="$BUNDLE_DIR" CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" CACTUS_TEST_SUITE="$SUITE" + export CACTUS_TEST_MODEL="$BUNDLE_DIR" CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" CACTUS_TEST_SUITE="$SUITE" CACTUS_TEST_BACKEND="$BACKEND" exec "$(pwd)/tests/ios/run.sh" fi if [ "$ANDROID_MODE" = true ]; then - export CACTUS_TEST_MODEL="$BUNDLE_DIR" CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" CACTUS_TEST_SUITE="$SUITE" + export CACTUS_TEST_MODEL="$BUNDLE_DIR" CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" CACTUS_TEST_SUITE="$SUITE" CACTUS_TEST_BACKEND="$BACKEND" exec "$(pwd)/tests/android/run.sh" fi @@ -45,6 +47,7 @@ echo "Model: $CACTUS_TEST_MODEL" echo "Bundle: $BUNDLE_DIR" echo "Transcription model: $CACTUS_TEST_TRANSCRIPTION_MODEL" echo "Transcription bundle: $TRANSCRIPTION_BUNDLE_DIR" +echo "Backend: $BACKEND" cd "$PROJECT_ROOT/cactus-engine/tests" rm -rf build && mkdir build && cd build @@ -55,6 +58,7 @@ export CACTUS_TEST_MODEL="$BUNDLE_DIR" export CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" export CACTUS_TEST_ASSETS="$ASSETS_DIR" export CACTUS_INDEX_PATH="$ASSETS_DIR" +export CACTUS_TEST_BACKEND="$BACKEND" FAILED=0 if [ -n "$SUITE" ]; then diff --git a/cactus-engine/tests/run.cpp b/cactus-engine/tests/run.cpp index 359e4bd3c..58d47ce67 100644 --- a/cactus-engine/tests/run.cpp +++ b/cactus-engine/tests/run.cpp @@ -742,7 +742,7 @@ std::string build_messages(const std::string& system_prompt, void print_usage(const char* argv0) { std::cerr << "Usage: " << argv0 - << " [--system ] [--image ] [--audio ]" + << " [--backend cpu|metal] [--system ] [--image ] [--audio ]" << " [--prompt ] [--input-ids ] [--input-ids-file ] [--max-new-tokens ]" << " [--result-json ] [--thinking] [--no-cloud-handoff]" << " [--confidence-threshold ] [--cloud-timeout-ms ] [-h|--help]\n"; @@ -772,6 +772,7 @@ int main(int argc, char** argv) { bool auto_handoff = true; double confidence_threshold = -1.0; int cloud_timeout_ms = 15000; + std::string backend; int i = 2; auto need_value = [&](const char* flag) -> bool { @@ -821,6 +822,9 @@ int main(int argc, char** argv) { } else if (arg == "--cloud-timeout-ms") { if (!need_value("--cloud-timeout-ms")) return 1; cloud_timeout_ms = std::max(0, std::atoi(argv[++i])); + } else if (arg == "--backend") { + if (!need_value("--backend")) return 1; + backend = argv[++i]; } else { std::cerr << "Error: unknown option '" << arg << "'\n"; print_usage(argv[0]); @@ -851,6 +855,16 @@ int main(int argc, char** argv) { } } + if (backend == "cpu" || backend == "metal") { + if (cactus_set_backend(backend.c_str()) == 0) + std::cout << "Backend: " << (backend == "metal" ? "Metal GPU" : "CPU") << "\n"; + else + std::cout << "Metal not available; using CPU\n"; + } else if (!backend.empty() && backend != "auto") { + std::cerr << "Error: unknown backend '" << backend << "' (expected 'cpu', 'metal', or 'auto')\n"; + return 1; + } + cactus_log_set_callback(log_callback, nullptr); std::cout << "Loading model from " << model_path << "...\n"; diff --git a/cactus-engine/tests/test_llm.cpp b/cactus-engine/tests/test_llm.cpp index 58cb1eaad..93ce6d9b2 100644 --- a/cactus-engine/tests/test_llm.cpp +++ b/cactus-engine/tests/test_llm.cpp @@ -437,7 +437,7 @@ struct LengthTokenizer : public Tokenizer { bool test_tool_constraint_clear_releases_bias() { LengthTokenizer tok; ToolCallConstrainer constrainer; - constrainer.init(Config::ModelType::GEMMA4, {{"get_weather", {"location"}, {"location"}}}, &tok); + constrainer.init(Config::ModelType::GEMMA4, {{"get_weather", {"location"}, {}, {"location"}}}, &tok); if (constrainer.get_bias().empty()) { std::cerr << " expected bias after activating init\n"; return false; diff --git a/cactus-engine/tests/test_needle.cpp b/cactus-engine/tests/test_needle.cpp index 905672a09..4f114e8b3 100644 --- a/cactus-engine/tests/test_needle.cpp +++ b/cactus-engine/tests/test_needle.cpp @@ -90,6 +90,7 @@ bool test_needle_tool_call() { } int main() { + TestUtils::apply_backend(); bool ok = true; ok &= test_needle_tool_call(); std::cout << "\n" << (ok ? "✓ needle tool-call test passed" : "✗ needle tool-call test failed") << "\n"; diff --git a/cactus-engine/tests/test_utils.cpp b/cactus-engine/tests/test_utils.cpp index 955095622..1dd50fcf4 100644 --- a/cactus-engine/tests/test_utils.cpp +++ b/cactus-engine/tests/test_utils.cpp @@ -1,10 +1,25 @@ #include "test_utils.h" #include +#include +#include namespace TestUtils { +void apply_backend() { + static bool applied = false; + if (applied) return; + applied = true; + const char* b = std::getenv("CACTUS_TEST_BACKEND"); + if (!b || !*b || std::strcmp(b, "auto") == 0) return; + if (cactus_set_backend(b) == 0) + std::cout << "Backend: " << (std::strcmp(b, "metal") == 0 ? "Metal GPU" : "CPU") << "\n"; + else + std::cout << "Backend '" << b << "' unavailable; using default\n"; +} + TestRunner::TestRunner(const std::string& suite_name) : suite_name_(suite_name), passed_count_(0), total_count_(0) { + apply_backend(); std::cout << "\n╔══════════════════════════════════════════════════════════════════════════════════════╗\n" << "║ Running " << std::left << std::setw(76) << suite_name_ << " ║\n" << "╚══════════════════════════════════════════════════════════════════════════════════════╝\n"; diff --git a/cactus-engine/tests/test_utils.h b/cactus-engine/tests/test_utils.h index 19aba3819..04cb792fe 100644 --- a/cactus-engine/tests/test_utils.h +++ b/cactus-engine/tests/test_utils.h @@ -27,6 +27,8 @@ class TestRunner { int total_count_; }; +void apply_backend(); + } namespace EngineTestUtils { diff --git a/cactus-graph/CMakeLists.txt b/cactus-graph/CMakeLists.txt index c82b62a02..9c431a855 100644 --- a/cactus-graph/CMakeLists.txt +++ b/cactus-graph/CMakeLists.txt @@ -12,6 +12,8 @@ add_library(cactus_graph STATIC src/core.cpp src/builder.cpp src/execute.cpp + src/metal_runtime.cpp + src/metal_plan.cpp src/io.cpp src/param_io.cpp src/ops_math.cpp diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index b14f824d4..4d08a1ef9 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -17,6 +17,9 @@ #include #include +int cactus_backend_select(const char* backend); +bool cactus_backend_metal(); + namespace cactus { enum class LogLevel { @@ -81,7 +84,7 @@ class Logger { #define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) #define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) -enum class ComputeBackend { CPU, NPU }; +enum class ComputeBackend { CPU, NPU, METAL }; enum class Activation { SILU, GELU, GELU_ERF, RELU, SIGMOID, TANH }; @@ -452,10 +455,37 @@ namespace GraphFile { struct SerializedGraph; } +struct FusedEmbedCtx { + int token_id = -1, position = 0; + CactusQuantMatrix ple{}; + CactusQuantMatrix proj{}; + const void* rms_weight = nullptr; + float emb_scale = 0.0f, ple_scale = 0.0f, proj_scale = 0.0f, final_scale = 0.0f, rms_eps = 1e-6f; + bool ok = false; +}; +void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx); +const FusedEmbedCtx* cactus_graph_fused_embed(); +bool cactus_graph_metal_fold_prologue(void* h_buf, void* ple_buf, void* pos_buf, + const CactusQuantMatrix* lm_head, size_t nl, size_t ple_dim); +bool cactus_graph_metal_tail(void* logits, size_t vocab); +void cactus_graph_metal_tail_commit(); + +bool cactus_graph_metal_argmax(uint32_t* idx, float* best, float* second); +void cactus_graph_mark_unadjusted(); +void cactus_graph_set_prefill_consistent(bool on); +bool cactus_graph_prefill_consistent(); +void cactus_graph_on_destroy(const void* graph); +void cactus_graph_set_sampling(const uint32_t* recent, int n_recent, float rep_penalty, + const float* bias_dense, size_t bias_len, + long long suppressed); +void cactus_graph_clear_sampling(); +bool cactus_graph_metal_adjusted(); +bool cactus_graph_metal_argmax_biased(); + class CactusGraph { public: CactusGraph(); - ~CactusGraph() = default; + ~CactusGraph(); CactusGraph(const CactusGraph&) = delete; CactusGraph& operator=(const CactusGraph&) = delete; CactusGraph(CactusGraph&&) noexcept = default; @@ -705,6 +735,7 @@ class CactusGraph { void invalidate_persistent(size_t persistent_node_id); void execute(const std::string& profile_file = ""); + bool extract_ple_pathway(FusedEmbedCtx& ctx) const; void hard_reset(); void soft_reset(); void soft_reset_keep_pool(); @@ -730,6 +761,7 @@ class CactusGraph { const std::vector& backup); void allocate_buffers(); size_t get_node_count() const; + void prewarm_metal_quant_weights(); void set_runtime_input_shape(size_t node_id, const std::vector& shape); void set_input_dynamic_dims(size_t node_id, const std::vector& dynamic_dims); bool has_dynamic_shapes() const { return has_dynamic_shapes_; } @@ -756,6 +788,15 @@ class CactusGraph { std::unordered_set populated_node_ids_; std::unordered_set embedded_input_node_ids_; std::unordered_set retained_output_node_ids_; + void build_metal_retype_plan(); + void invalidate_metal_state(); + std::unordered_map metal_plans_; + std::unordered_map> metal_plan_banned_; + uint64_t metal_plan_sig_ = 0; + std::vector metal_retype_plan_; + bool metal_retype_built_ = false; + bool metal_retype_disabled_ = false; + std::unordered_map> metal_persistent_acts_; }; namespace GraphFile { diff --git a/cactus-graph/src/execute.cpp b/cactus-graph/src/execute.cpp index fb1481cbb..eb5a1ccba 100644 --- a/cactus-graph/src/execute.cpp +++ b/cactus-graph/src/execute.cpp @@ -1,5 +1,6 @@ #include "../cactus_graph.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include #include #include @@ -13,6 +14,20 @@ #include #include +static int g_backend_override = -1; +int cactus_backend_select(const char* backend) { + if (!backend) return -1; + if (std::strcmp(backend, "auto") == 0) { g_backend_override = -1; return 0; } + if (std::strcmp(backend, "metal") == 0) { + if (!cactus_metal_available()) return -1; + g_backend_override = 1; + return 0; + } + if (std::strcmp(backend, "cpu") == 0) { g_backend_override = 0; return 0; } + return -1; +} +bool cactus_backend_metal() { return g_backend_override != 0 && cactus_metal_available(); } + using ComputeFn = void(*)(GraphNode&, const nodes_vector&, const node_index_map_t&); #define DECLARE_COMPUTE(name) \ @@ -429,7 +444,899 @@ void CactusGraph::infer_shapes() { runtime_shapes_dirty_ = false; } +static void row_strides(const std::vector& shape, size_t* out) { + size_t s=1; for (int k=(int)shape.size()-1; k>=0; --k){ out[k]=s; s*=shape[k]; } +} +static void bcast_strides(const std::vector& in_shape, const std::vector& out_shape, uint32_t* out) { + size_t off = out_shape.size() - in_shape.size(); + size_t istr[8]; row_strides(in_shape, istr); + for (size_t d=0; d dequantize_int8_weights_to_fp16(const BufferDesc& W, size_t rows, size_t cols, + const char* op_name); + +static const __fp16* metal_conv_weight_f16(const BufferDesc& w, size_t rows, size_t cols) { + if (w.precision == Precision::FP16) return w.data_as<__fp16>(); + if (w.precision != Precision::INT8) return nullptr; + const int8_t* p = w.data_as(); + size_t n = rows * cols; + if (!p || n == 0 || w.total_size < n) return nullptr; + uint64_t h = 1469598103934665603ull; + size_t take = n < 64 ? n : 64; + for (size_t i = 0; i < take; ++i) { h ^= (uint8_t)p[i]; h *= 1099511628211ull; } + for (size_t i = n - take; i < n; ++i) { h ^= (uint8_t)p[i]; h *= 1099511628211ull; } + h ^= n; h *= 1099511628211ull; + struct Entry { std::vector<__fp16> data; uint64_t fp; }; + static std::mutex mu; + static std::unordered_map cache; + std::lock_guard lk(mu); + Entry& e = cache[p]; + if (e.data.size() != n || e.fp != h) { + e.data = dequantize_int8_weights_to_fp16(w, rows, cols, "conv_metal"); + e.fp = h; + } + return e.data.data(); +} + +static bool try_encode_metal(GraphNode& node, const nodes_vector& nodes, const node_index_map_t& map) { + BufferDesc& out = node.output_buffer; + auto fp16 = [](const BufferDesc& b){ return b.precision == Precision::FP16; }; + switch (node.op_type) { + case OpType::MATMUL: { + const auto& lhs = get_input(node, 0, nodes, map); + const auto& rhs = get_input(node, 1, nodes, map); + size_t M = lhs.shape[lhs.shape.size() - 2]; + if (!fp16(lhs)) return false; + if (PrecisionTraits::is_cq(rhs.precision) && rhs.group_size > 0) { + CactusQuantMatrix mat = rhs.to_cq_matrix(); + if (M == 1 && !cactus_graph_prefill_consistent()) + return cactus_metal_encode_quant_matmul(out.get_data(), lhs.get_data(), &mat); + return cactus_metal_encode_quant_matmul_m(out.get_data(), lhs.get_data(), &mat, (uint32_t)M); + } + if (!fp16(rhs) || !fp16(out) || rhs.shape.size() != 2) return false; + size_t K = lhs.shape.back(); + size_t N = node.params.pretransposed_rhs ? rhs.shape[0] : rhs.shape[1]; + size_t rhs_k = node.params.pretransposed_rhs ? rhs.shape[1] : rhs.shape[0]; + if (K == 0 || rhs_k != K || lhs.total_size % K != 0) return false; + size_t rows = lhs.total_size / K; + if (rows != M) { + size_t batch = rows / M; + if (batch * M != rows) return false; + } + return cactus_metal_encode_gemm_f16(out.get_data(), lhs.get_data(), rhs.get_data(), + (uint32_t)rows, (uint32_t)K, (uint32_t)N, node.params.pretransposed_rhs ? 1 : 0); + } + case OpType::ADD: case OpType::ADD_CLIPPED: case OpType::SUBTRACT: + case OpType::MULTIPLY: case OpType::DIVIDE: case OpType::NOT_EQUAL: { + const auto& a = get_input(node, 0, nodes, map); + const auto& b = get_input(node, 1, nodes, map); + int code = node.op_type==OpType::ADD?0: node.op_type==OpType::ADD_CLIPPED?1: + node.op_type==OpType::SUBTRACT?2: node.op_type==OpType::MULTIPLY?3: + node.op_type==OpType::DIVIDE?4:5; + bool f32 = a.precision == Precision::FP32 && b.precision == Precision::FP32 + && out.precision == Precision::FP32; + if (f32 && a.total_size == out.total_size && b.total_size == out.total_size) + return cactus_metal_encode_binary_f32(code, out.get_data(), a.get_data(), b.get_data(), out.total_size); + if (!fp16(a) || !fp16(b) || !fp16(out)) return false; + if (a.total_size == out.total_size && b.total_size == out.total_size) + return cactus_metal_encode_binary(code, out.get_data(), a.get_data(), b.get_data(), out.total_size); + const auto& osh = out.shape; + uint32_t nd = (uint32_t)osh.size(); + if (nd == 0 || nd > 8 || a.shape.size() > nd || b.shape.size() > nd) return false; + uint32_t oshape[8], astr[8], bstr[8]; + for (uint32_t d=0; d 8 || in.shape.size() != nd || out.shape.size() != nd) return false; + bool tailswap = nd >= 2; + for (uint32_t d = 0; d + 2 < nd && tailswap; ++d) if (perm[d] != d) tailswap = false; + if (tailswap && (perm[nd-2] != nd-1 || perm[nd-1] != nd-2)) tailswap = false; + if (tailswap) { + size_t batch = 1; + for (uint32_t d = 0; d + 2 < nd; ++d) batch *= in.shape[d]; + return cactus_metal_encode_transpose2d(out.get_data(), in.get_data(), + (uint32_t)batch, (uint32_t)in.shape[nd-2], (uint32_t)in.shape[nd-1]); + } + size_t istr[8]; row_strides(in.shape, istr); + uint32_t oshape[8], sstride[8]; + for (uint32_t d=0; d 8 || axis >= nd || out.shape.size() != nd) return false; + size_t istr[8]; row_strides(in.shape, istr); + uint32_t oshape[8], sstride[8]; + for (uint32_t d=0; d= in.shape.size()) return false; + size_t istr[8]; row_strides(in.shape, istr); + size_t slice = istr[axis], block = istr[axis-1]; + uint32_t oshape[2] = { (uint32_t)(in.total_size/block), (uint32_t)slice }; + uint32_t sstride[2] = { (uint32_t)block, 1u }; + return cactus_metal_encode_strided_copy(out.get_data(), in.get_data(), oshape, sstride, 2, + (uint32_t)out.total_size, (uint32_t)(node.params.index_value * slice), in.byte_size, out.byte_size); + } + case OpType::CAT: { + if (node.input_ids.size() < 2 || out.shape.empty()) return false; + size_t axis = (size_t)node.params.axis; + uint32_t nd = (uint32_t)out.shape.size(); + if (axis >= nd || nd > 8) return false; + for (size_t ii = 0; ii < node.input_ids.size(); ++ii) { + const auto& cin = get_input(node, ii, nodes, map); + if (!fp16(cin) || cin.shape.size() != nd) return false; + } + size_t ostr[8]; row_strides(out.shape, ostr); + size_t axis_off = 0; + for (size_t ii = 0; ii < node.input_ids.size(); ++ii) { + const auto& cin = get_input(node, ii, nodes, map); + uint32_t ishape[8], ostride[8]; + for (uint32_t d=0; d 0 && cin.shape[0] == 1 && out.shape[0] > 1) ? out.shape[0] : 1; + for (size_t r = 0; r < bcast; ++r) { + if (!cactus_metal_encode_strided_scatter(out.get_data(), cin.get_data(), ishape, ostride, nd, + (uint32_t)cin.total_size, (uint32_t)(axis_off * ostr[axis] + r * ostr[0]), cin.byte_size, out.byte_size)) + return false; + } + axis_off += cin.shape[axis]; + } + return true; + } + case OpType::SCALAR_ADD: case OpType::SCALAR_SUBTRACT: + case OpType::SCALAR_MULTIPLY: case OpType::SCALAR_DIVIDE: + case OpType::SCALAR_EXP: case OpType::SCALAR_SQRT: case OpType::SCALAR_COS: + case OpType::SCALAR_SIN: case OpType::SCALAR_LOG: case OpType::ABS: + case OpType::POW: case OpType::SCALAR_NOT_EQUAL: case OpType::LEAKY_RELU: { + const auto& in = get_input(node, 0, nodes, map); + int code; + switch (node.op_type) { + case OpType::SCALAR_ADD: code = 0; break; + case OpType::SCALAR_SUBTRACT: code = 1; break; + case OpType::SCALAR_MULTIPLY: code = 2; break; + case OpType::SCALAR_DIVIDE: code = 3; break; + case OpType::SCALAR_EXP: code = 4; break; + case OpType::SCALAR_SQRT: code = 5; break; + case OpType::SCALAR_COS: code = 6; break; + case OpType::SCALAR_SIN: code = 7; break; + case OpType::SCALAR_LOG: code = 8; break; + case OpType::ABS: code = 9; break; + case OpType::POW: code = 10; break; + case OpType::SCALAR_NOT_EQUAL: code = 11; break; + default: code = 12; break; + } + if (in.precision == Precision::FP32 && out.precision == Precision::FP32) + return cactus_metal_encode_scalar_f32(code, out.get_data(), in.get_data(), out.total_size, node.params.scalar); + if (!fp16(in) || !fp16(out)) return false; + return cactus_metal_encode_scalar(code, out.get_data(), in.get_data(), out.total_size, node.params.scalar); + } + case OpType::GELU: case OpType::TANH: case OpType::SILU: case OpType::RELU: + case OpType::GELU_ERF: case OpType::SIGMOID: { + const auto& in = get_input(node, 0, nodes, map); + int code = node.op_type==OpType::GELU?0: node.op_type==OpType::TANH?1: node.op_type==OpType::SILU?2: + node.op_type==OpType::GELU_ERF?4: node.op_type==OpType::SIGMOID?5:3; + if (in.precision == Precision::FP32 && out.precision == Precision::FP32) + return cactus_metal_encode_unary_f32(code, out.get_data(), in.get_data(), out.total_size); + if (!fp16(in) || !fp16(out)) return false; + return cactus_metal_encode_unary(code, out.get_data(), in.get_data(), out.total_size); + } + case OpType::SUM: case OpType::MEAN: case OpType::VARIANCE: + case OpType::MIN: case OpType::MAX: { + const auto& in = get_input(node, 0, nodes, map); + int axis = node.params.axis; + if (axis < 0 || (size_t)axis >= in.shape.size()) return false; + bool f32 = in.precision == Precision::FP32 && out.precision == Precision::FP32; + if (!f32 && (!fp16(in) || !fp16(out))) return false; + int code = node.op_type==OpType::SUM?0: node.op_type==OpType::MEAN?1: + node.op_type==OpType::VARIANCE?2: node.op_type==OpType::MIN?3:4; + size_t outer = 1, inner = 1; + for (size_t d = 0; d < (size_t)axis; ++d) outer *= in.shape[d]; + for (size_t d = (size_t)axis + 1; d < in.shape.size(); ++d) inner *= in.shape[d]; + return cactus_metal_encode_reduce_axis(code, out.get_data(), in.get_data(), + (uint32_t)outer, (uint32_t)in.shape[(size_t)axis], (uint32_t)inner, f32 ? 1 : 0); + } + case OpType::CUMSUM: { + const auto& in = get_input(node, 0, nodes, map); + int axis = node.params.axis; + if (axis < 0 || (size_t)axis >= in.shape.size()) return false; + bool f32 = in.precision == Precision::FP32 && out.precision == Precision::FP32; + if (!f32 && (!fp16(in) || !fp16(out))) return false; + size_t outer = 1, inner = 1; + for (size_t d = 0; d < (size_t)axis; ++d) outer *= in.shape[d]; + for (size_t d = (size_t)axis + 1; d < in.shape.size(); ++d) inner *= in.shape[d]; + return cactus_metal_encode_cumsum(out.get_data(), in.get_data(), + (uint32_t)outer, (uint32_t)in.shape[(size_t)axis], (uint32_t)inner, f32 ? 1 : 0); + } + case OpType::CONCAT: { + if (node.input_ids.size() != 2) return false; + const auto& a = get_input(node, 0, nodes, map); + const auto& b = get_input(node, 1, nodes, map); + if (!fp16(a) || !fp16(b) || !fp16(out)) return false; + int axis = node.params.axis; + if (axis < 0) axis += (int)a.shape.size(); + if (axis < 0 || (size_t)axis >= a.shape.size() || a.shape.size() != b.shape.size()) return false; + size_t a_outer = 1, b_outer = 1, inner = 1; + for (size_t d = 0; d < (size_t)axis; ++d) { a_outer *= a.shape[d]; b_outer *= b.shape[d]; } + for (size_t d = (size_t)axis + 1; d < a.shape.size(); ++d) inner *= a.shape[d]; + return cactus_metal_encode_concat2(out.get_data(), a.get_data(), b.get_data(), + (uint32_t)a_outer, (uint32_t)b_outer, + (uint32_t)a.shape[(size_t)axis], (uint32_t)b.shape[(size_t)axis], (uint32_t)inner); + } + case OpType::GATHER: { + const auto& t = get_input(node, 0, nodes, map); + const auto& idx = get_input(node, 1, nodes, map); + if (!fp16(t) || !fp16(out) || idx.precision != Precision::FP32) return false; + if (t.shape.size() < 2) return false; + size_t D = t.total_size / t.shape[0]; + return cactus_metal_encode_gather_f32idx(out.get_data(), t.get_data(), idx.get_data(), + (uint32_t)idx.total_size, (uint32_t)D, t.byte_size); + } + case OpType::ROPE: case OpType::ROPE_GPTJ: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || !fp16(out) || in.shape.size() < 4) return false; + size_t S = in.shape[1], H = in.shape[2], D = in.shape[3]; + bool gptj = node.op_type == OpType::ROPE_GPTJ; + size_t rot = gptj ? (size_t)node.params.scalar : D; + if (rot == 0 || rot > D || (rot % 2) != 0) return false; + size_t tokens = in.total_size / D; + return cactus_metal_encode_rope_full(out.get_data(), in.get_data(), + (uint32_t)tokens, (uint32_t)S, (uint32_t)H, (uint32_t)D, (uint32_t)rot, + (uint32_t)node.params.position_offset, node.params.theta, gptj ? 1 : 0); + } + case OpType::MAXPOOL1D: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || !fp16(out) || in.shape.size() != 3 || out.shape.size() != 3) return false; + return cactus_metal_encode_maxpool1d(out.get_data(), in.get_data(), + (uint32_t)(in.shape[0] * in.shape[1]), (uint32_t)in.shape[2], + (uint32_t)out.shape[2], (uint32_t)node.params.kernel_size, (uint32_t)node.params.stride); + } + case OpType::BILINEAR_INTERPOLATION: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || !fp16(out) || in.shape.size() != 2) return false; + size_t src = (size_t)std::sqrt((double)in.shape[0]); + if (src * src != in.shape[0]) return false; + return cactus_metal_encode_bilinear(out.get_data(), in.get_data(), + (uint32_t)src, (uint32_t)src, (uint32_t)node.params.dst_height, + (uint32_t)node.params.dst_width, (uint32_t)in.shape[1], + node.params.align_corners ? 1 : 0); + } + case OpType::CONV1D: case OpType::CONV1D_K7S3: { + if (node.input_ids.size() < 2) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const BufferDesc* b = node.input_ids.size() > 2 ? &get_input(node, 2, nodes, map) : nullptr; + if (!fp16(x) || !fp16(w) || !fp16(out)) return false; + if (b && b->precision != Precision::FP16) return false; + if (x.shape.size() != 3 || w.shape.size() != 3 || out.shape.size() != 3) return false; + bool ck_co = node.op_type == OpType::CONV1D_K7S3; + size_t K = ck_co ? w.shape[1] : w.shape[2]; + size_t Cout = ck_co ? w.shape[2] : w.shape[0]; + return cactus_metal_encode_conv1d_gen(out.get_data(), x.get_data(), w.get_data(), + b ? b->get_data() : nullptr, (uint32_t)x.shape[0], (uint32_t)x.shape[1], + (uint32_t)x.shape[2], (uint32_t)Cout, (uint32_t)out.shape[2], (uint32_t)K, + (uint32_t)(node.params.stride ? node.params.stride : 1), ck_co ? 1 : 0); + } + case OpType::CONV1D_CAUSAL: case OpType::CONV1D_SAME_DEPTHWISE_K9: { + if (node.input_ids.size() < 2) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + bool causal = node.op_type == OpType::CONV1D_CAUSAL; + const BufferDesc* b = (!causal && node.input_ids.size() > 2) ? &get_input(node, 2, nodes, map) : nullptr; + if (!fp16(x) || !fp16(out)) return false; + if (b && b->precision != Precision::FP16) return false; + if (x.shape.size() != 3) return false; + size_t C = x.shape[2]; + size_t K; + if (causal) { + if (w.shape.size() != 3 || w.shape[0] != C || w.shape[1] != 1) return false; + K = w.shape[2]; + } else { + K = w.shape.back(); + if (w.total_size != C * K) return false; + } + const __fp16* wptr = metal_conv_weight_f16(w, C, K); + if (!wptr) return false; + size_t dil = causal ? (node.params.dilation ? node.params.dilation : 1) : 1; + size_t pad = causal ? (K - 1) * dil : K / 2; + return cactus_metal_encode_conv1d_nlc_dw(out.get_data(), x.get_data(), wptr, + b ? b->get_data() : nullptr, (uint32_t)x.shape[0], (uint32_t)x.shape[1], + (uint32_t)C, (uint32_t)K, (uint32_t)dil, (uint32_t)pad); + } + case OpType::CONV2D_K3S2P1: case OpType::CONV2D_K3S1P1: + case OpType::CONV2D_DEPTHWISE_K3S2P1: case OpType::CONV2D_POINTWISE_1X1: { + if (node.input_ids.size() < 2) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const BufferDesc* b = node.input_ids.size() > 2 ? &get_input(node, 2, nodes, map) : nullptr; + if (!fp16(x) || !fp16(w) || !fp16(out)) return false; + if (b && b->precision != Precision::FP16) return false; + if (x.shape.size() != 4 || out.shape.size() != 4) return false; + bool dw = node.op_type == OpType::CONV2D_DEPTHWISE_K3S2P1; + bool pw = node.op_type == OpType::CONV2D_POINTWISE_1X1; + uint32_t K = pw ? 1 : 3; + uint32_t stride = (node.op_type == OpType::CONV2D_K3S1P1 || pw) ? 1 : 2; + uint32_t pad = pw ? 0 : 1; + return cactus_metal_encode_conv2d(out.get_data(), x.get_data(), w.get_data(), + b ? b->get_data() : nullptr, (uint32_t)x.shape[0], (uint32_t)x.shape[1], + (uint32_t)x.shape[2], (uint32_t)x.shape[3], (uint32_t)out.shape[1], + (uint32_t)out.shape[2], (uint32_t)out.shape[3], K, stride, pad, dw ? 1 : 0); + } + case OpType::CONV1D_POINTWISE: { + if (node.input_ids.size() < 2) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const BufferDesc* b = node.input_ids.size() > 2 ? &get_input(node, 2, nodes, map) : nullptr; + if (!fp16(x) || !fp16(out)) return false; + if (b && b->precision != Precision::FP16) return false; + if (x.shape.size() != 3 || out.shape.size() != 3) return false; + size_t Cin = x.shape[2], Cout = out.shape[2]; + if (w.total_size != Cin * Cout) return false; + const __fp16* wptr = metal_conv_weight_f16(w, Cout, Cin); + if (!wptr) return false; + size_t rows = x.shape[0] * x.shape[1]; + if (!cactus_metal_encode_gemm_f16(out.get_data(), x.get_data(), wptr, + (uint32_t)rows, (uint32_t)Cin, (uint32_t)Cout, 1)) return false; + if (b) return cactus_metal_encode_bias_add_rows(out.get_data(), b->get_data(), + (uint32_t)Cout, (uint32_t)out.total_size); + return true; + } + case OpType::REL_POS_BIAS: { + if (node.input_ids.size() != 2) return false; + const auto& q = get_input(node, 0, nodes, map); + const auto& r = get_input(node, 1, nodes, map); + if (!fp16(q) || !fp16(r) || !fp16(out)) return false; + if (q.shape.size() != 4 || r.shape.size() != 4) return false; + size_t B = q.shape[0], T = q.shape[1], H = q.shape[2], D = q.shape[3]; + size_t Rb = r.shape[0], R = r.shape[1]; + if (B == 0 || T == 0 || (Rb != 1 && Rb != B)) return false; + if (r.shape[2] != H || r.shape[3] != D) return false; + if (R < 2 * T - 1 || out.total_size != B * H * T * T) return false; + return cactus_metal_encode_rel_pos_bias(out.get_data(), q.get_data(), r.get_data(), + (uint32_t)B, (uint32_t)T, (uint32_t)H, (uint32_t)D, (uint32_t)R, + Rb == 1 ? 0 : 1, node.params.scale); + } + case OpType::BATCHNORM: { + if (node.input_ids.size() != 5) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const auto& b = get_input(node, 2, nodes, map); + const auto& rm = get_input(node, 3, nodes, map); + const auto& rv = get_input(node, 4, nodes, map); + if (!fp16(x) || !fp16(out) || !fp16(w) || !fp16(b) || !fp16(rm) || !fp16(rv)) return false; + int axis = node.params.axis; + if (axis < 0) axis += (int)x.shape.size(); + if (axis < 0 || (size_t)axis >= x.shape.size()) return false; + size_t inner = 1; + for (size_t d = (size_t)axis + 1; d < x.shape.size(); ++d) inner *= x.shape[d]; + return cactus_metal_encode_batchnorm(out.get_data(), x.get_data(), w.get_data(), + b.get_data(), rm.get_data(), rv.get_data(), (uint32_t)x.shape[(size_t)axis], + (uint32_t)inner, (uint32_t)x.total_size, node.params.epsilon); + } + case OpType::GROUPNORM: { + if (node.input_ids.size() < 3) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const auto& b = get_input(node, 2, nodes, map); + if (!fp16(x) || !fp16(out) || !fp16(w) || !fp16(b) || x.shape.size() < 2) return false; + size_t spatial = 1; + for (size_t d = 2; d < x.shape.size(); ++d) spatial *= x.shape[d]; + size_t groups = node.params.num_groups ? node.params.num_groups : 32; + return cactus_metal_encode_groupnorm(out.get_data(), x.get_data(), w.get_data(), + b.get_data(), (uint32_t)x.shape[0], (uint32_t)x.shape[1], (uint32_t)spatial, + (uint32_t)groups, node.params.epsilon); + } + case OpType::CLAMP: { + const auto& in = get_input(node, 0, nodes, map); + bool f32 = in.precision == Precision::FP32 && out.precision == Precision::FP32; + if (!f32 && (!fp16(in) || !fp16(out))) return false; + return cactus_metal_encode_clamp(out.get_data(), in.get_data(), out.total_size, + node.params.scalar, node.params.scale, f32 ? 1 : 0); + } + case OpType::GLU: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || !fp16(out) || out.shape.empty()) return false; + int axis = node.params.axis; + if (axis < 0) axis += (int)in.shape.size(); + if (axis < 0 || (size_t)axis >= in.shape.size()) return false; + size_t split = in.shape[(size_t)axis] / 2; + size_t inner = 1; + for (size_t i = (size_t)axis + 1; i < in.shape.size(); ++i) inner *= in.shape[i]; + return cactus_metal_encode_glu(out.get_data(), in.get_data(), split, inner, out.total_size); + } + case OpType::SOFTMAX: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || !fp16(out) || out.shape.empty()) return false; + size_t cols = out.shape.back(); + if (cols == 0) return false; + return cactus_metal_encode_softmax_rows(out.get_data(), in.get_data(), out.total_size / cols, cols); + } + case OpType::GATED_DELTANET_DECODE: { + if (node.input_ids.size() != 6) return false; + const auto& q = get_input(node, 0, nodes, map); + const auto& k = get_input(node, 1, nodes, map); + const auto& v = get_input(node, 2, nodes, map); + const auto& g = get_input(node, 3, nodes, map); + const auto& b = get_input(node, 4, nodes, map); + const auto& st = get_input(node, 5, nodes, map); + if (!fp16(q) || !fp16(k) || !fp16(v) || !fp16(g) || !fp16(b) || !fp16(st) || !fp16(out)) return false; + if (q.shape.size() != 4 || v.shape.size() != 4 || st.shape.size() != 4) return false; + if (q.shape[1] != 1) return false; + size_t B = q.shape[0], Hq = q.shape[2], K = q.shape[3]; + size_t Hv = v.shape[2], V = v.shape[3]; + if (Hq == 0 || (Hv % Hq) != 0) return false; + if (st.shape[0] != B || st.shape[1] != K || st.shape[2] != Hv || st.shape[3] != V) return false; + if (out.total_size != B * (1 + K) * Hv * V) return false; + return cactus_metal_encode_deltanet_decode(out.get_data(), q.get_data(), k.get_data(), + v.get_data(), g.get_data(), b.get_data(), st.get_data(), + (uint32_t)B, (uint32_t)Hq, (uint32_t)Hv, (uint32_t)K, (uint32_t)V, node.params.scale); + } + case OpType::GATED_DELTANET_PREFILL: { + if (node.input_ids.size() != 6) return false; + const auto& q = get_input(node, 0, nodes, map); + const auto& k = get_input(node, 1, nodes, map); + const auto& v = get_input(node, 2, nodes, map); + const auto& g = get_input(node, 3, nodes, map); + const auto& b = get_input(node, 4, nodes, map); + const auto& st = get_input(node, 5, nodes, map); + if (!fp16(q) || !fp16(k) || !fp16(v) || !fp16(g) || !fp16(b) || !fp16(st) || !fp16(out)) return false; + if (q.shape.size() != 4 || v.shape.size() != 4 || st.shape.size() != 4) return false; + size_t B = q.shape[0], T = q.shape[1], Hq = q.shape[2], K = q.shape[3]; + size_t Hv = v.shape[2], V = v.shape[3]; + if (Hq == 0 || (Hv % Hq) != 0) return false; + if (st.shape[0] != B || st.shape[1] != K || st.shape[2] != Hv || st.shape[3] != V) return false; + if (out.total_size != B * (T + K) * Hv * V) return false; + return cactus_metal_encode_deltanet_prefill(out.get_data(), q.get_data(), k.get_data(), + v.get_data(), g.get_data(), b.get_data(), st.get_data(), + (uint32_t)B, (uint32_t)T, (uint32_t)Hq, (uint32_t)Hv, (uint32_t)K, (uint32_t)V, node.params.scale); + } + case OpType::RECURRENT_CACHE_WRITE: { + if (node.input_ids.size() < 2) return false; + const auto& src = get_input(node, 0, nodes, map); + BufferDesc& cache = nodes[map.at(node.input_ids[1])]->output_buffer; + if (!src.get_data() || !cache.get_data() || src.byte_size == 0) return false; + if (src.byte_size > cache.byte_size) return false; + return cactus_metal_encode_copy(cache.get_data(), src.get_data(), src.byte_size); + } + case OpType::TOPK: { + const auto& in = get_input(node, 0, nodes, map); + if (!fp16(in) || out.precision != Precision::FP32) return false; + if (in.shape.size() != 2 || node.params.top_k == 0 || node.params.top_k > 16) return false; + return cactus_metal_encode_topk_rows(out.get_data(), in.get_data(), + in.shape[0], in.shape[1], node.params.top_k); + } + case OpType::MOE_LAYER: { + const size_t num_experts = node.params.num_experts; + const size_t top_k = node.params.num_experts_per_tok; + if (!node.params.moe_gated || num_experts == 0 || top_k == 0 || top_k > 16) return false; + if (node.input_ids.size() != 3 + 3 * num_experts) return false; + const auto& hidden = get_input(node, 0, nodes, map); + const auto& probs = get_input(node, 1, nodes, map); + const auto& topk = get_input(node, 2, nodes, map); + if (!fp16(hidden) || !fp16(out) || !fp16(probs)) return false; + if (topk.precision != Precision::FP32) return false; + if (hidden.shape.size() != 2 || probs.shape.size() != 2 || probs.shape[1] != num_experts) return false; + uint32_t act; + switch (node.params.activation) { + case Activation::SILU: act = 0u; break; + case Activation::GELU: act = 1u; break; + default: return false; + } + const auto& w1_0 = get_input(node, 3, nodes, map); + if (!PrecisionTraits::is_cq(w1_0.precision) || w1_0.group_size == 0) return false; + CactusQuantMatrix key = w1_0.to_cq_matrix(); + if (!cactus_metal_moe_cq4_ready(&key)) { + std::vector w1s(num_experts), w3s(num_experts), w2s(num_experts); + for (size_t e = 0; e < num_experts; ++e) { + const auto& b1 = get_input(node, 3 + e, nodes, map); + const auto& b3 = get_input(node, 3 + num_experts + e, nodes, map); + const auto& b2 = get_input(node, 3 + 2 * num_experts + e, nodes, map); + if (!PrecisionTraits::is_cq(b1.precision) || b1.group_size == 0 || + !PrecisionTraits::is_cq(b3.precision) || b3.group_size == 0 || + !PrecisionTraits::is_cq(b2.precision) || b2.group_size == 0) return false; + w1s[e] = b1.to_cq_matrix(); + w3s[e] = b3.to_cq_matrix(); + w2s[e] = b2.to_cq_matrix(); + } + if (!cactus_metal_moe_cq4_build(w1s.data(), w3s.data(), w2s.data(), (uint32_t)num_experts)) + return false; + } + return cactus_metal_encode_moe_gated_cq4(out.get_data(), hidden.get_data(), probs.get_data(), + topk.get_data(), &key, (uint32_t)num_experts, (uint32_t)top_k, (uint32_t)hidden.shape[0], + act, node.params.normalize_routing ? 1u : 0u, node.params.epsilon, node.params.scalar); + } + case OpType::LAYERNORM: { + if (node.input_ids.size() < 2 || out.shape.empty()) return false; + const auto& in = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + const BufferDesc* b = node.input_ids.size() > 2 ? &get_input(node, 2, nodes, map) : nullptr; + if (!fp16(in) || !fp16(out) || !fp16(w) || (b && b->precision != Precision::FP16)) return false; + size_t dim = out.shape.back(); + if (dim == 0 || w.total_size != dim) return false; + return cactus_metal_encode_layer_norm(out.get_data(), in.get_data(), w.get_data(), + b ? b->get_data() : nullptr, out.total_size / dim, dim, node.params.epsilon); + } + case OpType::CONV1D_K3: { + if (node.input_ids.size() < 2) return false; + const auto& x = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + if (!fp16(x) || !fp16(out)) return false; + if (x.shape.size() != 3 || w.shape.size() != 3 || x.shape[0] != 1 || w.shape[2] != 3) return false; + size_t Cin = x.shape[1], L = x.shape[2], Cout = w.shape[0]; + size_t stride = node.params.stride ? node.params.stride : 1; + size_t Lout = ((L - 1) / stride) + 1; + int w_int8 = w.precision == Precision::INT8 ? 1 : 0; + if (!w_int8 && w.precision != Precision::FP16) return false; + if (out.total_size != Cout * Lout) return false; + return cactus_metal_encode_conv1d_k3(out.get_data(), x.get_data(), w.get_data(), w_int8, + w.activation_scales_data, (uint32_t)w.group_size, + (uint32_t)Cin, (uint32_t)L, (uint32_t)Cout, (uint32_t)Lout, (uint32_t)stride); + } + case OpType::ATTENTION: { + if (node.input_ids.size() < 3) return false; + const auto& q = get_input(node, 0, nodes, map); + const auto& k = get_input(node, 1, nodes, map); + const auto& v = get_input(node, 2, nodes, map); + const BufferDesc* mk = node.input_ids.size() > 3 ? &get_input(node, 3, nodes, map) : nullptr; + if (!fp16(q) || !fp16(k) || !fp16(v) || !fp16(out)) return false; + if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) return false; + size_t B = q.shape[0], T = q.shape[1], HQ = q.shape[2], D = q.shape[3]; + size_t S = k.shape[1], HKV = k.shape[2], DV = v.shape[3]; + if (HKV == 0 || HQ % HKV != 0) return false; + uint32_t mask_mode = 0; + if (mk) { + if (mk->precision != Precision::FP16) return false; + bool per_head = mk->shape.size() == 4; + if (!per_head && mk->shape.size() != 3) return false; + mask_mode = (node.params.attention_mask_is_additive ? 1u : 2u) + (per_head ? 2u : 0u); + } + float scale = node.params.scale != 0.0f ? node.params.scale : 1.0f/std::sqrt((float)D); + return cactus_metal_encode_attention_f16(out.get_data(), q.get_data(), k.get_data(), v.get_data(), + mk ? mk->get_data() : nullptr, + (uint32_t)B, (uint32_t)T, (uint32_t)S, (uint32_t)HQ, (uint32_t)HKV, (uint32_t)D, (uint32_t)DV, + scale, node.params.is_causal ? 1u : 0u, (uint32_t)node.params.position_offset, + (uint32_t)node.params.window_size, node.params.logit_cap, mask_mode); + } + case OpType::RMS_NORM: { + if (node.input_ids.size() < 2 || out.shape.empty()) return false; + const auto& in = get_input(node, 0, nodes, map); + const auto& w = get_input(node, 1, nodes, map); + if (!fp16(in) || !fp16(out) || !fp16(w)) return false; + size_t dim = out.shape.back(); + if (dim == 0) return false; + return cactus_metal_encode_rms_norm(out.get_data(), in.get_data(), w.get_data(), + out.total_size / dim, dim, node.params.epsilon); + } + case OpType::PRECISION_CAST: { + const auto& in = get_input(node, 0, nodes, map); + return cactus_metal_encode_cast(out.get_data(), static_cast(out.precision), + in.get_data(), static_cast(in.precision), in.total_size); + } + case OpType::ATTENTION_CACHED: { + if (node.input_ids.size() < 5) return false; + const auto& qb = get_input(node, 0, nodes, map); + const auto& knew = get_input(node, 1, nodes, map); + const auto& vnew = get_input(node, 2, nodes, map); + const auto& kcache = get_input(node, 3, nodes, map); + const auto& vcache = get_input(node, 4, nodes, map); + if (qb.shape.size() < 3) return false; + size_t batch=qb.shape[0], seq=qb.shape[1], nqh=qb.shape[2]; + if (batch != 1) return false; + if (kcache.precision == Precision::FP16 || vcache.precision == Precision::FP16) return false; + const uint64_t* km = reinterpret_cast(kcache.get_data()); + const uint64_t* vm = reinterpret_cast(vcache.get_data()); + if (!km || !vm) return false; + size_t cache_len=km[0], max_seq=km[1], kv_heads=km[2], hdim=km[3], v_max=vm[1]; + if (kv_heads == 0 || hdim == 0 || nqh % kv_heads != 0) return false; + size_t v_hdim = node.params.v_head_dim > 0 ? node.params.v_head_dim : hdim; + size_t new_seq_len = knew.total_size / (kv_heads * hdim); + size_t history_len = cache_len >= new_seq_len ? cache_len - new_seq_len : 0; + size_t po = node.params.position_offset, pos, new_len = seq; + if (po == std::numeric_limits::max()) pos = history_len; + else if (po == std::numeric_limits::max() - 1) { + pos = (cache_len >= seq) ? cache_len - seq : 0; history_len = cache_len; new_len = 0; + } else pos = po; + size_t total_keys = history_len + new_len; + size_t win = node.params.window_size; + size_t kv_start = (win > 0 && pos > win) ? pos - win : 0; + if (pos > history_len) kv_start = 0; + size_t kv_end = node.params.is_causal ? std::min(total_keys, pos + 1) : total_keys; + float scale = node.params.scale != 0.0f ? node.params.scale : 1.0f/std::sqrt((float)hdim); + const char* bk = static_cast(kcache.get_data()); + const char* bv = static_cast(vcache.get_data()); + size_t ngK=(hdim+31)/32, ngV=(v_hdim+31)/32; + if (seq > 1) { + size_t sink = km[4]; + uint32_t ringv = (win > 0 && max_seq > 2*sink + 1) ? (uint32_t)(max_seq - 2*sink - 1) : 0u; + return cactus_metal_encode_attention_i8_prefill( + out.get_data(), qb.get_data(), knew.get_data(), vnew.get_data(), + bk + 64, bv + 64, + bk + 64 + max_seq*kv_heads*hdim, bv + 64 + v_max*kv_heads*v_hdim, + (uint32_t)nqh, (uint32_t)kv_heads, (uint32_t)hdim, (uint32_t)v_hdim, + (uint32_t)history_len, (uint32_t)new_len, (uint32_t)pos, + (uint32_t)win, node.params.is_causal ? 1u : 0u, (uint32_t)seq, scale, + max_seq*kv_heads*hdim, v_max*kv_heads*v_hdim, + max_seq*kv_heads*ngK*sizeof(float), v_max*kv_heads*ngV*sizeof(float), + (uint32_t)sink, ringv); + } + size_t cache_ceiling = nodes[map.at(node.input_ids[3])]->params.max_cache_seq_len; + if (win > 0 && win < cache_ceiling) { + size_t sink = km[4]; + size_t ringW = max_seq > sink + 1 ? max_seq - sink - 1 : 0; + if (ringW == 0) return false; + if (total_keys > ringW) { + history_len = ringW; + total_keys = ringW; + kv_start = 0; + kv_end = ringW; + } + } + bool ok = cactus_metal_encode_attention_i8( + out.get_data(), qb.get_data(), knew.get_data(), vnew.get_data(), + bk + 64, bv + 64, + bk + 64 + max_seq*kv_heads*hdim, bv + 64 + v_max*kv_heads*v_hdim, + (uint32_t)nqh, (uint32_t)kv_heads, (uint32_t)hdim, (uint32_t)v_hdim, + (uint32_t)history_len, (uint32_t)total_keys, (uint32_t)kv_start, (uint32_t)kv_end, scale, + history_len*kv_heads*hdim, history_len*kv_heads*v_hdim, + history_len*kv_heads*ngK*sizeof(float), history_len*kv_heads*ngV*sizeof(float)); + return ok; + } + case OpType::KV_CACHE_APPEND: { + if (node.input_ids.size() < 2) return false; + const auto& new_kv = get_input(node, 0, nodes, map); + GraphNode& cache_node = *nodes[map.at(node.input_ids[1])]; + BufferDesc& cache = cache_node.output_buffer; + if (!fp16(new_kv) || cache.precision == Precision::FP16 || !cache.get_data()) return false; + uint64_t* km = reinterpret_cast(cache.get_data()); + size_t current_len=km[0], max_len=km[1], kv_heads=km[2], hdim=km[3], sink=km[4], num_slots=km[5]; + if (num_slots != 1 || kv_heads == 0 || hdim == 0) return false; + size_t new_seq_len = new_kv.total_size / (kv_heads * hdim); + size_t ceiling = cache_node.params.max_cache_seq_len, ws = node.params.window_size; + bool sliding = ws > 0 && ws < ceiling; + size_t new_total = current_len + new_seq_len; + char* base = static_cast(cache.get_data()); + size_t ngK = (hdim + 31)/32; + if (new_seq_len > 1) { + if (sliding) { + if (max_len <= sink + 1) return false; + uint32_t W = (uint32_t)(max_len - sink - 1); + if (!cactus_metal_encode_kv_append_ring_i8_m(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)current_len, 32, (uint32_t)new_seq_len, (uint32_t)sink, W, + new_kv.byte_size, max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = new_total; + } else if (ws > 0 && new_total > max_len) { + size_t window = max_len; + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) return false; + size_t remaining = std::min(tail_capacity - new_seq_len, current_len - keep_sink); + size_t shift_src = current_len - remaining; + if (!cactus_metal_encode_kv_append_sliding_i8_m(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)keep_sink, (uint32_t)remaining, (uint32_t)shift_src, 32, (uint32_t)new_seq_len, + new_kv.byte_size, max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = keep_sink + remaining + new_seq_len; + } else { + if (new_total > max_len) { + if (!cactus_kv_cache_grow(cache, new_total, ceiling)) return false; + km = reinterpret_cast(cache.get_data()); + max_len = km[1]; base = static_cast(cache.get_data()); + if (new_total > max_len) return false; + } + if (!cactus_metal_encode_kv_append_i8_m(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)current_len, 32, (uint32_t)new_seq_len, + new_kv.byte_size, max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = new_total; + } + if (out.get_data()) *out.data_as() = static_cast(km[0]); + return true; + } + if (sliding) { + if (max_len <= sink + 1) return false; + uint32_t W = (uint32_t)(max_len - sink - 1); + if (!cactus_metal_encode_kv_append_ring_i8_m(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)current_len, 32, 1u, (uint32_t)sink, W, + new_kv.byte_size, max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = new_total; + if (out.get_data()) *out.data_as() = static_cast(new_total); + return true; + } + if (new_total > max_len) { + if (!cactus_kv_cache_grow(cache, new_total, ceiling)) return false; + km = reinterpret_cast(cache.get_data()); + max_len = km[1]; base = static_cast(cache.get_data()); + if (new_total > max_len) return false; + } + size_t window = max_len; + if (new_total > window) { + size_t keep_sink = std::min({sink, current_len, window}); + size_t tail_capacity = window - keep_sink; + if (new_seq_len >= tail_capacity) return false; + size_t remaining = std::min(tail_capacity - new_seq_len, current_len - keep_sink); + size_t shift_src = current_len - remaining; + if (!cactus_metal_encode_kv_append_sliding_i8(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)keep_sink, (uint32_t)remaining, (uint32_t)shift_src, 32, new_kv.byte_size, + max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = keep_sink + remaining + new_seq_len; + if (out.get_data()) *out.data_as() = static_cast(km[0]); + return true; + } + if (!cactus_metal_encode_kv_append_i8(new_kv.get_data(), base + 64, + base + 64 + max_len*kv_heads*hdim, (uint32_t)kv_heads, (uint32_t)hdim, + (uint32_t)current_len, 32, new_kv.byte_size, + max_len*kv_heads*hdim, max_len*kv_heads*ngK*sizeof(float))) + return false; + km[0] = new_total; + if (out.get_data()) *out.data_as() = static_cast(new_total); + return true; + } + case OpType::CONV_CACHE_APPEND: { + if (node.input_ids.size() < 2) return false; + const auto& new_data = get_input(node, 0, nodes, map); + BufferDesc& cache = nodes[map.at(node.input_ids[1])]->output_buffer; + if (!fp16(out) || !cache.get_data() || !out.get_data()) return false; + if (new_data.precision != Precision::FP16 && new_data.precision != Precision::FP32) return false; + uint64_t* cm = reinterpret_cast(cache.get_data()); + uint64_t head = cm[0], count = cm[1], ws = cm[2], hd = cm[3]; + if (ws == 0 || hd == 0 || head >= ws) return false; + size_t num_rows = new_data.total_size / hd; + if (num_rows == 0 || out.total_size != ws * hd) return false; + uint32_t nnew = (uint32_t)std::min(num_rows, ws); + uint64_t count_new = std::min(ws, count + num_rows); + if (!cactus_metal_encode_conv_cache_append(out.get_data(), new_data.get_data(), + static_cast(cache.get_data()) + 64, + (uint32_t)hd, (uint32_t)ws, nnew, (uint32_t)head, (uint32_t)count_new, + (uint32_t)num_rows, new_data.precision == Precision::FP32 ? 1 : 0)) + return false; + cm[0] = (head + nnew) % ws; + cm[1] = count_new; + return true; + } + case OpType::EMBEDDING: { + if (node.input_ids.size() < 2) return false; + const auto& emb = get_input(node, 0, nodes, map); + const auto& idxb = get_input(node, 1, nodes, map); + if (!fp16(out)) return false; + size_t M = idxb.total_size; + if (M == 0 || M > 4096) return false; + std::vector rows(M); + if (idxb.precision == Precision::FP32) { const float* p = idxb.data_as(); for (size_t i=0;i(); for (size_t i=0;i(); for (size_t i=0;i 0) { + CactusQuantMatrix W = emb.to_cq_matrix(); + if (W.flags & CACTUS_QUANT_FLAG_ORTHOGONAL) + return cactus_metal_encode_embedding_ortho_m(out.get_data(), &W, rows.data(), (uint32_t)M); + return cactus_metal_encode_embedding_hadamard_m(out.get_data(), &W, rows.data(), (uint32_t)M); + } + if (emb.precision == Precision::FP16 && emb.shape.size() >= 2) { + uint32_t D = (uint32_t)emb.shape[1]; + return cactus_metal_encode_gather_f16(out.get_data(), emb.get_data(), + emb.total_size * sizeof(__fp16), rows.data(), (uint32_t)M, D); + } + return false; + } + default: return false; + } +} + +struct MetalFusePlan; +MetalFusePlan* cactus_metal_plan_build(const std::vector>& nodes, + const std::unordered_map& map, + const std::unordered_set& pinned_ids, + const std::vector& retype, + const std::unordered_set* banned = nullptr); +void cactus_metal_plan_free(MetalFusePlan* p); +bool cactus_metal_plan_fold(MetalFusePlan* p, const std::vector>& nodes); +const std::vector* cactus_metal_plan_exec_list(const MetalFusePlan* p); +void* cactus_metal_plan_arena_ptr(const MetalFusePlan* p, size_t i); +bool cactus_metal_plan_has_arena(const MetalFusePlan* p); +void cactus_metal_plan_extend_last_use(const MetalFusePlan* p, std::vector& last_use); +int32_t cactus_metal_plan_action(const MetalFusePlan* p, size_t i); +bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, + const std::vector>& nodes, + const std::unordered_map& map); + +void CactusGraph::build_metal_retype_plan() { + metal_retype_built_ = true; + const size_t n = nodes_.size(); + metal_retype_plan_.assign(n, 0); + auto idxof = [&](size_t id) -> long { + auto it = node_index_map_.find(id); + return it == node_index_map_.end() ? -1 : (long)it->second; + }; + std::vector> cons(n); + for (size_t i = 0; i < n; ++i) + for (size_t id : nodes_[i]->input_ids) { long j = idxof(id); if (j >= 0) cons[(size_t)j].push_back(i); } + auto interior = [&](const GraphNode& nd) { + if (nd.output_buffer.precision != Precision::FP32) return false; + if (nd.output_buffer.shape.size() > 8) return false; + switch (nd.op_type) { + case OpType::ADD: case OpType::ADD_CLIPPED: case OpType::SUBTRACT: + case OpType::MULTIPLY: case OpType::DIVIDE: + case OpType::SCALAR_ADD: case OpType::SCALAR_SUBTRACT: + case OpType::SCALAR_MULTIPLY: case OpType::SCALAR_DIVIDE: + case OpType::CLAMP: case OpType::GELU: case OpType::TANH: + case OpType::SILU: case OpType::RELU: + case OpType::VIEW: case OpType::RESHAPE: case OpType::FLATTEN: + case OpType::TRANSPOSE: case OpType::CAT: + return std::fabs(nd.params.scalar) < 60000.0f && std::fabs(nd.params.scale) < 60000.0f; + default: return false; + } + }; + std::vector cand(n, 0); + for (size_t i = 0; i < n; ++i) { + const GraphNode& nd = *nodes_[i]; + if (nd.op_type == OpType::PRECISION_CAST && nd.output_buffer.precision == Precision::FP32 + && !nd.input_ids.empty()) { + long j = idxof(nd.input_ids[0]); + if (j >= 0 && nodes_[(size_t)j]->output_buffer.precision == Precision::FP16) cand[i] = 1; + continue; + } + if (!interior(nd) || nd.input_ids.empty()) continue; + bool all = true; + for (size_t id : nd.input_ids) { + long j = idxof(id); + if (j < 0 || !cand[(size_t)j]) { all = false; break; } + } + cand[i] = all ? 1 : 0; + } + std::vector ok = cand; + for (size_t ri = n; ri-- > 0;) { + if (!ok[ri]) continue; + const GraphNode& nd = *nodes_[ri]; + if (retained_output_node_ids_.count(nd.id) || persistent_node_ids_.count(nd.id)) { ok[ri] = 0; continue; } + if (cons[ri].empty()) { ok[ri] = 0; continue; } + for (size_t c : cons[ri]) { + const GraphNode& cn = *nodes_[c]; + bool absorbs = (cn.op_type == OpType::PRECISION_CAST + && cn.output_buffer.precision == Precision::FP16) + || (cand[c] && ok[c]); + if (!absorbs) { ok[ri] = 0; break; } + } + } + for (size_t i = 0; i < n; ++i) { + if (!ok[i]) continue; + metal_retype_plan_[i] = nodes_[i]->op_type == OpType::PRECISION_CAST ? 1 : 2; + } + for (size_t i = 0; i < n; ++i) { + const GraphNode& nd = *nodes_[i]; + if (nd.op_type != OpType::PRECISION_CAST || nd.output_buffer.precision != Precision::FP16 + || nd.input_ids.empty()) continue; + long j = idxof(nd.input_ids[0]); + if (j >= 0 && metal_retype_plan_[(size_t)j] != 0) metal_retype_plan_[i] = 1; + } +} + void CactusGraph::execute(const std::string& profile_file) { + cactus_graph_mark_unadjusted(); BufferPool& pool = buffer_pool_; const size_t n = nodes_.size(); infer_shapes(); @@ -440,6 +1347,7 @@ void CactusGraph::execute(const std::string& profile_file) { }; bool trace_execution = get_env_int("CACTUS_TRACE_EXECUTE", 0) != 0; + static const size_t flush_cadence = (size_t)get_env_int("CACTUS_FLUSH_CADENCE", 48); bool trace_nan = get_env_int("CACTUS_TRACE_NAN", 0) != 0; bool need_debug = !profile_file.empty(); if (!need_debug) { @@ -502,15 +1410,78 @@ void CactusGraph::execute(const std::string& profile_file) { return true; }; - auto may_alias_input = [](const GraphNode& node) { - return (node.op_type == OpType::SLICE && node.params.axis == 0) || - (node.op_type == OpType::INDEX && node.params.axis == 0); + bool metal_mode = cactus_backend_metal(); + if (metal_mode && n < 100) { + for (size_t i = 0; i < n; ++i) { + OpType t = nodes_[i]->op_type; + if (t == OpType::LSTM_CELL || t == OpType::BILSTM_SEQUENCE) { + metal_mode = false; + break; + } + } + } + if (metal_mode && !need_debug && !metal_retype_built_) build_metal_retype_plan(); + MetalFusePlan* fplan = nullptr; + if (metal_mode && !need_debug) { + uint64_t sig = 1469598103934665603ull; + sig ^= (uint64_t)n; sig *= 1099511628211ull; + for (auto& np : nodes_) { + sig ^= (uint64_t)np->op_type + 0x9e3779b97f4a7c15ull; + sig *= 1099511628211ull; + if (np->op_type != OpType::INPUT) continue; + for (size_t d : np->output_buffer.shape) { + sig ^= (uint64_t)d + 0x9e3779b97f4a7c15ull; + sig *= 1099511628211ull; + } + } + auto it = metal_plans_.find(sig); + if (it == metal_plans_.end()) { + std::unordered_set pinned(retained_output_node_ids_.begin(), retained_output_node_ids_.end()); + pinned.insert(persistent_node_ids_.begin(), persistent_node_ids_.end()); + static const std::vector no_retype; + const std::vector& rt = (!metal_retype_disabled_ && metal_retype_plan_.size() == n) + ? metal_retype_plan_ : no_retype; + auto bit = metal_plan_banned_.find(sig); + const std::unordered_set* banned = + bit == metal_plan_banned_.end() ? nullptr : &bit->second; + it = metal_plans_.emplace(sig, cactus_metal_plan_build(nodes_, node_index_map_, pinned, rt, banned)).first; + } + fplan = it->second; + metal_plan_sig_ = sig; + } + const uint8_t* rplan = (metal_mode && !need_debug && !metal_retype_disabled_ + && metal_retype_plan_.size() == n) ? metal_retype_plan_.data() : nullptr; + auto plan_of = [&](const GraphNode& node) -> uint8_t { + if (!rplan) return 0; + auto it = node_index_map_.find(node.id); + return it == node_index_map_.end() ? 0 : rplan[it->second]; }; - - auto preallocates_output = [](const GraphNode& node) { - return !((node.op_type == OpType::SLICE && node.params.axis == 0) || - (node.op_type == OpType::INDEX && node.params.axis == 0)); + auto aliases_input = [&](const GraphNode& node) -> bool { + if (plan_of(node) == 1) return true; + if (node.op_type == OpType::SLICE && !node.input_ids.empty()) { + auto it = node_index_map_.find(node.input_ids[0]); + if (it != node_index_map_.end()) { + const auto& ish = nodes_[it->second]->output_buffer.shape; + size_t ax = static_cast(node.params.axis); + if (ax < ish.size()) { + size_t outer = 1; + for (size_t d = 0; d < ax; ++d) outer *= ish[d]; + if (outer == 1) return true; + } + } + } + if (node.op_type == OpType::INDEX && node.params.axis == 0) return true; + if (!metal_mode) return false; + if (node.op_type == OpType::VIEW || node.op_type == OpType::RESHAPE || node.op_type == OpType::FLATTEN) + return true; + if (node.op_type == OpType::PRECISION_CAST && !node.input_ids.empty()) { + auto it = node_index_map_.find(node.input_ids[0]); + if (it != node_index_map_.end() && + nodes_[it->second]->output_buffer.precision == node.output_buffer.precision) return true; + } + return false; }; + auto preallocates_output = [&](const GraphNode& node) { return !aliases_input(node); }; std::vector last_use(n, 0); std::vector use_count(n, 0); @@ -525,33 +1496,248 @@ void CactusGraph::execute(const std::string& profile_file) { } std::vector keep_until_graph_cleanup(n, false); - for (size_t i = 0; i < n; ++i) { + for (size_t i = n; i-- > 0;) { const auto& node = *nodes_[i]; - if (!may_alias_input(node) || node.input_ids.empty()) continue; + if (!aliases_input(node) || node.input_ids.empty()) continue; auto it = node_index_map_.find(node.input_ids[0]); if (it == node_index_map_.end()) continue; size_t base_idx = it->second; - if (use_count[i] == 0) { + if (use_count[i] == 0 || keep_until_graph_cleanup[i]) { keep_until_graph_cleanup[base_idx] = true; } else { last_use[base_idx] = std::max(last_use[base_idx], last_use[i]); } } + if (fplan) cactus_metal_plan_extend_last_use(fplan, last_use); + std::vector> release_after(n); for (size_t i = 0; i < n; ++i) { if (!can_release_node(i) || use_count[i] == 0 || keep_until_graph_cleanup[i]) continue; release_after[last_use[i]].push_back(i); } + if (metal_mode && !need_debug) { + const bool transient_acts = n >= 1500 && !cactus_metal_plan_has_arena(fplan); + std::vector transient_ptr(transient_acts ? n : 0, nullptr); + std::vector transient_dead; + std::vector transient_ok(transient_acts ? n : 0, 0); + auto release_transients = [&]() { + for (void* p : transient_dead) cactus_metal_free_shared(p); + transient_dead.clear(); + for (size_t i = 0; i < transient_ptr.size(); ++i) + if (transient_ptr[i]) { cactus_metal_free_shared(transient_ptr[i]); transient_ptr[i] = nullptr; } + }; + struct CacheWordSnap { size_t idx; size_t word; uint64_t value; }; + std::vector kv_snapshot; + auto kv_restore = [&]() { + for (const auto& s : kv_snapshot) { + uint64_t* km = reinterpret_cast(nodes_[s.idx]->output_buffer.get_data()); + if (km) km[s.word] = s.value; + } + }; + auto metal_abort_cleanup = [&]() { + cactus_metal_session_sync(); + kv_restore(); + release_transients(); + cactus_metal_set_active(false); + cactus_metal_session_end(); + }; + struct MetalExecGuard { + decltype(metal_abort_cleanup)& cleanup; + bool armed = true; + ~MetalExecGuard() { if (armed) cleanup(); } + } metal_guard{metal_abort_cleanup}; + cactus_metal_session_begin(); + cactus_metal_set_active(true); + if (transient_acts) { + for (size_t i = 0; i < n; ++i) { + size_t id = nodes_[i]->id; + transient_ok[i] = !retained_output_node_ids_.count(id) + && !persistent_node_ids_.count(id) + && !keep_until_graph_cleanup[i] + && nodes_[i]->output_buffer.byte_size >= (256u << 10); + } + } + auto metal_release = [&](size_t idx) { + GraphNode& nd = *nodes_[idx]; + if (aliases_input(nd)) nd.output_buffer.external_data = nullptr; + if (transient_acts && transient_ptr[idx]) { + transient_dead.push_back(transient_ptr[idx]); + transient_ptr[idx] = nullptr; + } + }; + auto assign_persistent_act = [&](GraphNode& nd) { + size_t need = nd.output_buffer.byte_size; + auto pit = metal_persistent_acts_.find(nd.id); + void* p = nullptr; + if (pit != metal_persistent_acts_.end() && pit->second.second >= need) { + p = pit->second.first; + } else { + if (pit != metal_persistent_acts_.end()) cactus_metal_free_shared(pit->second.first); + p = cactus_metal_alloc_pooled(need); + if (p) metal_persistent_acts_[nd.id] = { p, need }; + } + if (p) { nd.output_buffer.release_to_pool(pool); nd.output_buffer.set_external(p); } + else nd.output_buffer.resize_from_pool(pool); + }; + size_t since_flush = 0, since_recycle = 0; + if (fplan) cactus_metal_plan_fold(fplan, nodes_); + std::vector metal_live(n, 0); + auto maybe_recycle = [&]() { + if (!transient_acts || ++since_recycle < 256 || transient_dead.empty()) return; + for (void* p : transient_dead) cactus_metal_free_shared(p); + transient_dead.clear(); + cactus_metal_session_sync(); + std::fill(metal_live.begin(), metal_live.end(), 0); + since_flush = 0; + since_recycle = 0; + }; + for (size_t i = 0; i < n; ++i) { + const GraphNode& nd = *nodes_[i]; + bool kv = nd.op_type == OpType::KV_CACHE_APPEND; + bool conv = nd.op_type == OpType::CONV_CACHE_APPEND; + if ((!kv && !conv) || nd.input_ids.size() < 2) continue; + auto ci = node_index_map_.find(nd.input_ids[1]); + if (ci == node_index_map_.end()) continue; + const uint64_t* km = reinterpret_cast(nodes_[ci->second]->output_buffer.get_data()); + if (!km) continue; + kv_snapshot.push_back({ci->second, 0, km[0]}); + if (conv) kv_snapshot.push_back({ci->second, 1, km[1]}); + } + auto input_metal_live = [&](const GraphNode& nd) -> bool { + for (size_t id : nd.input_ids) { + auto it = node_index_map_.find(id); + if (it != node_index_map_.end() && metal_live[it->second]) return true; + } + return false; + }; + const std::vector* elist = fplan ? cactus_metal_plan_exec_list(fplan) : nullptr; + size_t iter_count = elist ? elist->size() : n; + if (fplan) { + for (size_t ii = 0; ii < iter_count; ++ii) { + size_t i = elist ? (size_t)(*elist)[ii] : ii; + if (cactus_metal_plan_action(fplan, i) == -3) assign_persistent_act(*nodes_[i]); + } + } + for (size_t ii = 0; ii < iter_count; ++ii) { + size_t i = elist ? (size_t)(*elist)[ii] : ii; + auto& node = nodes_[i]; + const OpType ot = node->op_type; + if (ot == OpType::INPUT) continue; + if (ot == OpType::KV_CACHE_STATE || ot == OpType::CONV_CACHE_STATE + || ot == OpType::RECURRENT_CACHE_STATE) { + dispatch_node(*node, nodes_, node_index_map_); + populated_node_ids_.insert(node->id); + for (size_t r : release_after[i]) metal_release(r); + continue; + } + int32_t fact = fplan ? cactus_metal_plan_action(fplan, i) : -1; + if (fact == -3) { + assign_persistent_act(*node); + metal_live[i] = 1; + for (size_t r : release_after[i]) metal_release(r); + continue; + } + if (aliases_input(*node)) { + if (rplan && rplan[i] == 1) { + const auto& src = get_input(*node, 0, nodes_, node_index_map_); + node->output_buffer.set_external(const_cast(static_cast(src.get_data()))); + } else { + dispatch_node(*node, nodes_, node_index_map_); + } + metal_live[i] = input_metal_live(*node) ? 1 : 0; + for (size_t r : release_after[i]) metal_release(r); + continue; + } + if (preallocates_output(*node)) { + void* ap = fplan ? cactus_metal_plan_arena_ptr(fplan, i) : nullptr; + if (ap) { + node->output_buffer.release_to_pool(pool); + node->output_buffer.set_external(ap); + } else if (transient_acts && transient_ok[i]) { + void* tp = cactus_metal_alloc_shared(node->output_buffer.byte_size); + if (tp) { + node->output_buffer.release_to_pool(pool); + node->output_buffer.set_external(tp); + transient_ptr[i] = tp; + } else { + assign_persistent_act(*node); + } + } else { + assign_persistent_act(*node); + } + } + if (fact >= 0) { + if (cactus_metal_plan_encode(fplan, fact, nodes_, node_index_map_)) { + metal_live[i] = 1; + for (size_t r : release_after[i]) metal_release(r); + if (++since_flush >= flush_cadence) { cactus_metal_session_flush(); since_flush = 0; } + maybe_recycle(); + continue; + } + metal_guard.armed = false; + metal_abort_cleanup(); + metal_plan_banned_[metal_plan_sig_].insert(i); + metal_plans_.erase(metal_plan_sig_); + cactus_metal_plan_free(fplan); + execute(profile_file); + return; + } + std::vector> flipped; + if (rplan && rplan[i] == 2) { + auto flip = [&](BufferDesc& b) { + if (b.precision == Precision::FP32) { flipped.push_back({&b, b.precision}); b.precision = Precision::FP16; } + }; + flip(node->output_buffer); + for (size_t id : node->input_ids) { + auto it = node_index_map_.find(id); + if (it != node_index_map_.end() && rplan[it->second] != 0) flip(nodes_[it->second]->output_buffer); + } + } + bool force_cpu = (ot == OpType::PRECISION_CAST + && node->output_buffer.total_size <= 8 && !input_metal_live(*node)) + || (ot == OpType::EMBEDDING && input_metal_live(*node)); + bool encoded = !force_cpu && try_encode_metal(*node, nodes_, node_index_map_); + for (auto& fp : flipped) fp.first->precision = fp.second; + if (!encoded && rplan && rplan[i] == 2) { + metal_retype_disabled_ = true; + metal_guard.armed = false; + metal_abort_cleanup(); + execute(profile_file); + return; + } + if (!encoded) { + if (input_metal_live(*node)) { + cactus_metal_session_sync(); + std::fill(metal_live.begin(), metal_live.end(), 0); + } + dispatch_node(*node, nodes_, node_index_map_); + } else { + metal_live[i] = 1; + } + if (ot == OpType::PERSISTENT) populated_node_ids_.insert(node->id); + for (size_t r : release_after[i]) metal_release(r); + if (encoded && ++since_flush >= flush_cadence) { cactus_metal_session_flush(); since_flush = 0; } + if (encoded) maybe_recycle(); + } + metal_guard.armed = false; + release_transients(); + cactus_metal_set_active(false); + cactus_metal_session_end(); + cactus_graph_metal_tail_commit(); + return; + } + if (!need_debug) { + auto run = [&](auto& nd) { dispatch_node(nd, nodes_, node_index_map_); }; for (size_t i = 0; i < n; ++i) { auto& node = nodes_[i]; if (node->op_type == OpType::INPUT) continue; if (node->op_type == OpType::KV_CACHE_STATE || node->op_type == OpType::CONV_CACHE_STATE || node->op_type == OpType::RECURRENT_CACHE_STATE) { - dispatch_node(*node, nodes_, node_index_map_); + run(*node); populated_node_ids_.insert(node->id); for (size_t release_idx : release_after[i]) { nodes_[release_idx]->output_buffer.release_memory(pool); @@ -561,7 +1747,7 @@ void CactusGraph::execute(const std::string& profile_file) { if (preallocates_output(*node)) { node->output_buffer.resize_from_pool(pool); } - dispatch_node(*node, nodes_, node_index_map_); + run(*node); trace_nonfinite(i, *node); if (node->op_type == OpType::PERSISTENT) { populated_node_ids_.insert(node->id); @@ -948,7 +2134,17 @@ void CactusGraph::execute(const std::string& profile_file) { } } +void CactusGraph::invalidate_metal_state() { + for (auto& kv : metal_plans_) cactus_metal_plan_free(kv.second); + metal_plans_.clear(); + for (auto& kv : metal_persistent_acts_) cactus_metal_free_shared(kv.second.first); + metal_persistent_acts_.clear(); + metal_retype_plan_.clear(); + metal_retype_built_ = false; +} + void CactusGraph::hard_reset() { + invalidate_metal_state(); nodes_.clear(); node_index_map_.clear(); mapped_files_.clear(); @@ -959,6 +2155,7 @@ void CactusGraph::hard_reset() { } void CactusGraph::soft_reset() { + invalidate_metal_state(); std::set cached_node_ids; for (const auto& cache_entry : weight_cache_) { cached_node_ids.insert(cache_entry.second); @@ -1000,6 +2197,7 @@ void CactusGraph::soft_reset() { } void CactusGraph::soft_reset_keep_pool() { + invalidate_metal_state(); std::set cached_node_ids; for (const auto& cache_entry : weight_cache_) { cached_node_ids.insert(cache_entry.second); @@ -1034,3 +2232,17 @@ void CactusGraph::soft_reset_keep_pool() { next_node_id_ = max_preserved_id + 1; debug_nodes_.clear(); } + +void CactusGraph::prewarm_metal_quant_weights() { + if (!cactus_backend_metal()) return; + for (auto& np : nodes_) { + GraphNode& node = *np; + if (node.op_type != OpType::MATMUL || node.input_ids.size() < 2) continue; + auto it = node_index_map_.find(node.input_ids[1]); + if (it == node_index_map_.end()) continue; + const BufferDesc& rhs = nodes_[it->second]->output_buffer; + if (!(PrecisionTraits::is_cq(rhs.precision) && rhs.group_size > 0)) continue; + CactusQuantMatrix mat = rhs.to_cq_matrix(); + cactus_metal_prewarm_quant(&mat); + } +} diff --git a/cactus-graph/src/metal_plan.cpp b/cactus-graph/src/metal_plan.cpp new file mode 100644 index 000000000..5a80117a1 --- /dev/null +++ b/cactus-graph/src/metal_plan.cpp @@ -0,0 +1,1819 @@ +#include "../cactus_graph.h" +#include "cactus_kernels.h" +#include "metal_backend.h" +#include +#include +#include + +bool cactus_kv_cache_grow(BufferDesc&, size_t, size_t); + +namespace { + +bool is_alias_op(OpType op) { + return op == OpType::VIEW || op == OpType::RESHAPE || op == OpType::FLATTEN; +} + +bool is_noop_transpose(const GraphNode& nd) { + if (nd.op_type != OpType::TRANSPOSE) return false; + size_t real = 0; + for (size_t d : nd.output_buffer.shape) if (d != 1) ++real; + return real <= 1; +} + +bool is_same_cast(const GraphNode& nd, const std::vector>& nodes, + const std::unordered_map& map) { + if (nd.op_type != OpType::PRECISION_CAST || nd.input_ids.empty()) return false; + auto it = map.find(nd.input_ids[0]); + return it != map.end() && nodes[it->second]->output_buffer.precision == nd.output_buffer.precision; +} + +} + +struct MetalCluster { + int rule = 0; + size_t a0 = 0, a1 = 0, a2 = 0, a3 = 0, a4 = 0, a5 = 0; + size_t b0 = 0, b1 = 0, b2 = 0, b3 = 0, b4 = 0; + float f0 = 0.0f, f1 = 0.0f; + uint32_t u0 = 0, u1 = 0; + void* s0 = nullptr; + void* s1 = nullptr; + void* sc[3] = {nullptr, nullptr, nullptr}; +}; + +struct MetalFusePlan { + std::vector action; + std::vector clusters; + long fold_h = -1, fold_ple = -1, fold_pos = -1, fold_w = -1; + size_t fold_nl = 0, fold_pd = 0; + std::vector exec_list; + std::vector arena_off; + char* arena_base = nullptr; + std::vector> blobs; +}; + +struct EwChainStep { + int32_t kind; + int32_t code; + float p0; + float p1; +}; + +MetalFusePlan* cactus_metal_plan_build( + const std::vector>& nodes, + const std::unordered_map& map, + const std::unordered_set& pinned_ids, + const std::vector& retype, + const std::unordered_set* banned) { + const size_t n = nodes.size(); + auto plan = new MetalFusePlan(); + plan->action.assign(n, -1); + std::vector pinned(n, 0); + for (size_t i = 0; i < n; ++i) + if (pinned_ids.count(nodes[i]->id)) pinned[i] = 1; + + auto idxof = [&](size_t id) -> long { + auto it = map.find(id); + return it == map.end() ? -1 : (long)it->second; + }; + std::vector> cons(n); + for (size_t i = 0; i < n; ++i) + for (size_t id : nodes[i]->input_ids) { long j = idxof(id); if (j >= 0) cons[(size_t)j].push_back(i); } + + auto retyped = [&](size_t idx) { return idx < retype.size() && retype[idx] != 0; }; + auto passthrough = [&](const GraphNode& nd) { + return is_alias_op(nd.op_type) || is_same_cast(nd, nodes, map) || is_noop_transpose(nd); + }; + auto up = [&](long i) -> long { + while (i >= 0) { + const GraphNode& nd = *nodes[(size_t)i]; + if (passthrough(nd) && !nd.input_ids.empty()) { + i = idxof(nd.input_ids[0]); + continue; + } + return i; + } + return -1; + }; + auto up_in = [&](size_t i, size_t k) -> long { + if (k >= nodes[i]->input_ids.size()) return -1; + return up(idxof(nodes[i]->input_ids[k])); + }; + auto deep_f16_source = [&](long i) -> long { + long cur = i; + while (cur >= 0) { + const GraphNode& nd = *nodes[(size_t)cur]; + if ((passthrough(nd) || nd.op_type == OpType::PRECISION_CAST) && !nd.input_ids.empty()) { + cur = idxof(nd.input_ids[0]); + continue; + } + break; + } + if (cur >= 0 && nodes[(size_t)cur]->output_buffer.precision == Precision::FP16) return cur; + return i; + }; + auto collect_chain = [&](size_t from, long to, std::vector& out_chain) { + long i = (long)from; + while (i >= 0 && i != to) { + const GraphNode& nd = *nodes[(size_t)i]; + if (!passthrough(nd)) break; + out_chain.push_back((size_t)i); + i = nd.input_ids.empty() ? -1 : idxof(nd.input_ids[0]); + } + }; + auto sole_use = [&](long i, const std::vector& allowed) -> bool { + if (i < 0) return false; + for (size_t c : cons[(size_t)i]) { + bool ok = false; + for (size_t a : allowed) if (a == c) { ok = true; break; } + if (!ok) { + const GraphNode& cn = *nodes[c]; + if (is_alias_op(cn.op_type) || is_same_cast(cn, nodes, map) || is_noop_transpose(cn)) { + bool subok = true; + for (size_t cc : cons[c]) { + bool found = false; + for (size_t a : allowed) if (a == cc) { found = true; break; } + if (!found) { subok = false; break; } + } + if (!subok) return false; + } else return false; + } + } + return true; + }; + + auto match_rope = [&](long add_i, long* x_out, long* cos_out, long* sin_out, + std::vector& cover) -> bool { + if (add_i < 0 || nodes[(size_t)add_i]->op_type != OpType::ADD) return false; + long m0 = up_in((size_t)add_i, 0), m1 = up_in((size_t)add_i, 1); + if (m0 < 0 || m1 < 0) return false; + if (nodes[(size_t)m0]->op_type != OpType::MULTIPLY) std::swap(m0, m1); + if (m0 < 0 || m1 < 0) return false; + if (nodes[(size_t)m0]->op_type != OpType::MULTIPLY || + nodes[(size_t)m1]->op_type != OpType::MULTIPLY) return false; + long rot = up_in((size_t)m1, 0); + long sinb = up_in((size_t)m1, 1); + if (rot < 0 || nodes[(size_t)rot]->op_type != OpType::CAT) { std::swap(rot, sinb); } + if (rot < 0 || sinb < 0 || nodes[(size_t)rot]->op_type != OpType::CAT) { + std::swap(m0, m1); + rot = up_in((size_t)m1, 0); sinb = up_in((size_t)m1, 1); + if (rot < 0 || nodes[(size_t)rot]->op_type != OpType::CAT) { std::swap(rot, sinb); } + if (rot < 0 || sinb < 0 || nodes[(size_t)rot]->op_type != OpType::CAT) return false; + } + long x = up_in((size_t)m0, 0), cosb = up_in((size_t)m0, 1); + if (x < 0 || cosb < 0) return false; + if (nodes[(size_t)rot]->input_ids.size() != 2) return false; + long neg = up_in((size_t)rot, 0), lo = up_in((size_t)rot, 1); + if (neg < 0 || lo < 0) return false; + if (nodes[(size_t)neg]->op_type != OpType::SCALAR_MULTIPLY || + nodes[(size_t)neg]->params.scalar != -1.0f) return false; + long hi = up_in((size_t)neg, 0); + if (hi < 0 || nodes[(size_t)hi]->op_type != OpType::SLICE || + nodes[(size_t)lo]->op_type != OpType::SLICE) return false; + long xs1 = up_in((size_t)hi, 0), xs2 = up_in((size_t)lo, 0); + if (xs1 != x || xs2 != x) { + long xa = up_in((size_t)m0, 1); + if (xs1 == xa && xs2 == xa) { x = xa; cosb = up_in((size_t)m0, 0); } + else return false; + } + *x_out = x; *cos_out = cosb; *sin_out = sinb; + cover.push_back((size_t)add_i); cover.push_back((size_t)m0); cover.push_back((size_t)m1); + cover.push_back((size_t)rot); cover.push_back((size_t)neg); + cover.push_back((size_t)hi); cover.push_back((size_t)lo); + return true; + }; + + auto alias_base = [&](long i, long* off_out) -> long { + long off = 0; + while (i >= 0) { + const GraphNode& nd = *nodes[(size_t)i]; + if (passthrough(nd) && !nd.input_ids.empty()) { + i = idxof(nd.input_ids[0]); + continue; + } + if (nd.op_type == OpType::INDEX && nd.params.axis == 0 && !nd.input_ids.empty()) { + long blk = 1; + for (size_t d : nd.output_buffer.shape) blk *= (long)d; + off += (long)nd.params.index_value * blk; + i = idxof(nd.input_ids[0]); + continue; + } + if (nd.op_type == OpType::SLICE && !nd.input_ids.empty()) { + long j = idxof(nd.input_ids[0]); + if (j < 0) break; + const auto& ish = nodes[(size_t)j]->output_buffer.shape; + size_t ax = (size_t)nd.params.axis; + if (ax >= ish.size()) break; + size_t outer = 1; + for (size_t d = 0; d < ax; ++d) outer *= ish[d]; + if (outer != 1) break; + long inner = 1; + for (size_t d = ax + 1; d < ish.size(); ++d) inner *= (long)ish[d]; + off += (long)nd.params.slice_start * inner; + i = j; + continue; + } + break; + } + *off_out = off; + return i; + }; + + auto release_scratch = [&](MetalCluster& c) { + if (c.s0) { cactus_metal_free_shared(c.s0); c.s0 = nullptr; } + if (c.s1) { cactus_metal_free_shared(c.s1); c.s1 = nullptr; } + for (auto& sp : c.sc) if (sp) { cactus_metal_free_shared(sp); sp = nullptr; } + }; + auto add_cluster = [&](MetalCluster c, size_t anchor, const std::vector& cover) -> bool { + if (banned && banned->count(anchor)) { release_scratch(c); return false; } + for (size_t v : cover) if (v != anchor && pinned[v]) { release_scratch(c); return false; } + int32_t cid = (int32_t)plan->clusters.size(); + plan->clusters.push_back(c); + for (size_t v : cover) if (v != anchor) plan->action[v] = -2; + plan->action[anchor] = cid; + return true; + }; + + struct AttnCand { + MetalCluster c; + size_t anchor; + std::vector cover; + size_t kcache; + long kapp, vapp; + }; + std::vector cands; + + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd = *nodes[i]; + + if (nd.op_type == OpType::ATTENTION_CACHED && nd.input_ids.size() >= 5) { + long qidx = idxof(nd.input_ids[0]); + if (qidx < 0) continue; + const BufferDesc& qb = nodes[(size_t)qidx]->output_buffer; + if (qb.shape.size() < 4 || qb.shape[0] != 1 || qb.shape[1] != 1) continue; + uint32_t nqh = (uint32_t)qb.shape[2], hd = (uint32_t)qb.shape[3]; + long qk = idxof(nd.input_ids[1]); + long vk = idxof(nd.input_ids[2]); + long kcache = idxof(nd.input_ids[3]); + long vcache = idxof(nd.input_ids[4]); + if (qk < 0 || vk < 0 || kcache < 0 || vcache < 0) continue; + if (nodes[(size_t)kcache]->params.num_kv_heads != 1) continue; + + std::vector cover; + long q_rope = up_in(i, 0); + long qx = -1, qcos = -1, qsin = -1; + if (!match_rope(q_rope, &qx, &qcos, &qsin, cover)) continue; + long qnorm = qx; + if (qnorm < 0 || nodes[(size_t)qnorm]->op_type != OpType::RMS_NORM) continue; + long qraw = up_in((size_t)qnorm, 0); + long qw = up_in((size_t)qnorm, 1); + if (qraw < 0 || qw < 0) continue; + cover.push_back((size_t)qnorm); + + long k_rope = up(qk); + long kx = -1, kcos = -1, ksin = -1; + std::vector kcover; + long knorm = -1, kraw = -1, kw = -1, vnorm = -1, vraw = -1, vw = -1; + long kapp = -1, vapp = -1; + bool has_new = false; + if (match_rope(k_rope, &kx, &kcos, &ksin, kcover)) { + knorm = kx; + if (knorm >= 0 && nodes[(size_t)knorm]->op_type == OpType::RMS_NORM) { + kraw = up_in((size_t)knorm, 0); + kw = up_in((size_t)knorm, 1); + long vn = up(vk); + if (kraw >= 0 && kw >= 0 && vn >= 0 && nodes[(size_t)vn]->op_type == OpType::RMS_NORM) { + vnorm = vn; + vraw = up_in((size_t)vn, 0); + vw = up_in((size_t)vn, 1); + for (size_t c = 0; c < n; ++c) { + if (nodes[c]->op_type != OpType::KV_CACHE_APPEND || nodes[c]->input_ids.size() < 2) continue; + long src = up(idxof(nodes[c]->input_ids[0])); + long cache = idxof(nodes[c]->input_ids[1]); + if (src == k_rope && cache == kcache) kapp = (long)c; + if (src == (long)vnorm && cache == vcache) vapp = (long)c; + } + if (kapp >= 0 && vapp >= 0 && vraw >= 0 && vw >= 0) has_new = true; + } + } + } + bool shared = false; + if (has_new) { + for (auto& pc : cands) + if (pc.kapp == kapp || pc.kcache == (size_t)kcache) { shared = true; break; } + } + + AttnCand cand; + cand.anchor = i; + cand.kcache = (size_t)kcache; + cand.kapp = kapp; cand.vapp = vapp; + MetalCluster& c = cand.c; + c.rule = 1; + c.b0 = (size_t)qcos; c.b1 = (size_t)qsin; + c.b2 = (size_t)kcache; c.b3 = (size_t)vcache; + c.b4 = i; + c.a0 = (size_t)qraw; c.a3 = (size_t)deep_f16_source(qw); + c.f0 = nodes[(size_t)qnorm]->params.epsilon; + c.f1 = nd.params.scale != 0.0f ? nd.params.scale : 1.0f / std::sqrt((float)hd); + c.u0 = nqh; c.u1 = hd; + cand.cover = cover; + collect_chain((size_t)idxof(nd.input_ids[0]), q_rope, cand.cover); + if (has_new && !shared) { + c.a1 = (size_t)kraw; c.a2 = (size_t)vraw; + c.a4 = (size_t)deep_f16_source(kw); c.a5 = (size_t)deep_f16_source(vw); + c.u1 |= 0x10000u; + cand.cover.insert(cand.cover.end(), kcover.begin(), kcover.end()); + cand.cover.push_back((size_t)knorm); + cand.cover.push_back((size_t)vnorm); + cand.cover.push_back((size_t)kapp); + cand.cover.push_back((size_t)vapp); + collect_chain((size_t)idxof(nd.input_ids[1]), k_rope, cand.cover); + collect_chain((size_t)idxof(nd.input_ids[2]), vnorm, cand.cover); + collect_chain((size_t)idxof(nodes[(size_t)kapp]->input_ids[0]), k_rope, cand.cover); + collect_chain((size_t)idxof(nodes[(size_t)vapp]->input_ids[0]), vnorm, cand.cover); + } else if (!shared) { + continue; + } + cands.push_back(std::move(cand)); + continue; + } + + if (nd.op_type == OpType::MATMUL && nd.input_ids.size() >= 2) { + long r = idxof(nd.input_ids[1]); + const BufferDesc* rb = r >= 0 ? &nodes[(size_t)r]->output_buffer : nullptr; + size_t M = 1; + for (size_t d = 0; d + 1 < nd.output_buffer.shape.size(); ++d) M *= nd.output_buffer.shape[d]; + if (rb && M == 1 && PrecisionTraits::is_cq(rb->precision) && rb->group_size > 0 + && PrecisionTraits::cq_bits(rb->precision) == 4 + && !(rb->cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) { + long mul = up_in(i, 0); + if (mul >= 0 && nodes[(size_t)mul]->op_type == OpType::MULTIPLY && sole_use(mul, {i})) { + long g0 = up_in((size_t)mul, 0), u0v = up_in((size_t)mul, 1); + for (int side = 0; side < 2; ++side) { + long gel = side == 0 ? g0 : u0v; + long other = side == 0 ? u0v : g0; + long sm = -1; + float sc_val = 1.0f; + if (gel >= 0 && nodes[(size_t)gel]->op_type == OpType::SCALAR_MULTIPLY + && sole_use(gel, {(size_t)mul})) { + sm = gel; + sc_val = nodes[(size_t)gel]->params.scalar; + gel = up_in((size_t)gel, 0); + } + if (gel >= 0 && other >= 0 && nodes[(size_t)gel]->op_type == OpType::GELU + && sole_use(gel, {sm >= 0 ? (size_t)sm : (size_t)mul})) { + long gate = up_in((size_t)gel, 0); + if (gate < 0) break; + if (sc_val == 1.0f && nodes[(size_t)gate]->op_type == OpType::MATMUL + && nodes[(size_t)gate]->input_ids.size() >= 2 + && sole_use(gate, {(size_t)gel})) { + long gr = idxof(nodes[(size_t)gate]->input_ids[1]); + const BufferDesc* grb = gr >= 0 ? &nodes[(size_t)gr]->output_buffer : nullptr; + size_t gM = 1; + for (size_t d = 0; d + 1 < nodes[(size_t)gate]->output_buffer.shape.size(); ++d) + gM *= nodes[(size_t)gate]->output_buffer.shape[d]; + if (grb && gM == 1 && PrecisionTraits::is_cq(grb->precision) + && grb->group_size == 128 && PrecisionTraits::cq_bits(grb->precision) == 4 + && !(grb->cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) + && cactus_metal_transform_gemv_fits((uint32_t)( + grb->shape.size() >= 2 ? grb->shape[1] : grb->shape[0]))) { + long gx = up_in((size_t)gate, 0); + if (gx >= 0) { + MetalCluster c7; + c7.rule = 7; + c7.a0 = (size_t)gx; + c7.a1 = (size_t)other; + c7.b0 = (size_t)gate; + c7.b4 = i; + c7.s0 = cactus_metal_alloc_shared(nodes[(size_t)gate]->output_buffer.total_size * 2); + std::vector cover; + cover.push_back((size_t)mul); + cover.push_back((size_t)gel); + cover.push_back((size_t)gate); + collect_chain((size_t)idxof(nd.input_ids[0]), mul, cover); + collect_chain((size_t)idxof(nodes[(size_t)gel]->input_ids[0]), gate, cover); + const GraphNode& rn = *nodes[(size_t)other]; + if (rn.op_type == OpType::INDEX && !rn.input_ids.empty() + && sole_use(other, {(size_t)mul})) { + long ib = idxof(rn.input_ids[0]); + if (ib >= 0) { + size_t rowlen = rn.output_buffer.total_size; + c7.a1 = (size_t)ib; + c7.u1 = (uint32_t)(rn.params.index_value * rowlen * 2); + c7.u0 = 1u; + cover.push_back((size_t)other); + } + } + add_cluster(c7, i, cover); + break; + } + } + } + MetalCluster c; + c.rule = 4; + c.a0 = (size_t)gate; + c.a1 = (size_t)other; + c.b4 = i; + c.f0 = sc_val; + const BufferDesc& gb = nodes[(size_t)gate]->output_buffer; + c.s0 = cactus_metal_alloc_shared(gb.total_size * 2); + std::vector cover; + cover.push_back((size_t)mul); + cover.push_back((size_t)gel); + if (sm >= 0) cover.push_back((size_t)sm); + collect_chain((size_t)idxof(nd.input_ids[0]), mul, cover); + add_cluster(c, i, cover); + break; + } + } + if (plan->action[i] >= 0) continue; + } + } + } + + if (nd.op_type == OpType::SCALAR_MULTIPLY && !nd.output_buffer.shape.empty() + && nd.output_buffer.shape.back() >= 32768 + && nd.output_buffer.total_size == nd.output_buffer.shape.back()) { + long th = up_in(i, 0); + if (th >= 0 && nodes[(size_t)th]->op_type == OpType::TANH) { + long dv = up_in((size_t)th, 0); + if (dv >= 0 && nodes[(size_t)dv]->op_type == OpType::SCALAR_DIVIDE + && nodes[(size_t)dv]->params.scalar == nd.params.scalar) { + long mm = up_in((size_t)dv, 0); + if (mm >= 0 && nodes[(size_t)mm]->op_type == OpType::MATMUL + && nodes[(size_t)mm]->input_ids.size() >= 2) { + long wnode = idxof(nodes[(size_t)mm]->input_ids[1]); + long xn = up_in((size_t)mm, 0); + if (wnode >= 0 && xn >= 0) { + const BufferDesc& wb = nodes[(size_t)wnode]->output_buffer; + if (PrecisionTraits::is_cq(wb.precision) && wb.group_size > 0 + && (wb.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) + && sole_use(mm, {(size_t)dv}) && sole_use(dv, {(size_t)th}) + && sole_use(th, {i})) { + MetalCluster c; + c.rule = 5; + c.a0 = (size_t)xn; c.a1 = (size_t)wnode; c.b4 = i; + c.f0 = nd.params.scalar; + c.s0 = cactus_metal_alloc_shared(wb.shape.empty() ? 0 : wb.to_cq_matrix().K * 2); + std::vector cover; + cover.push_back((size_t)mm); + cover.push_back((size_t)dv); + cover.push_back((size_t)th); + collect_chain((size_t)idxof(nodes[(size_t)dv]->input_ids[0]), mm, cover); + collect_chain((size_t)idxof(nd.input_ids[0]), th, cover); + add_cluster(c, i, cover); + continue; + } + } + } + } + } + } + + if (nd.op_type == OpType::CAT && nd.input_ids.size() >= 4 && !nd.output_buffer.shape.empty()) { + size_t cnt = nd.input_ids.size(); + bool blocky = true; + { + size_t ax = (size_t)nd.params.axis; + const auto& osh = nd.output_buffer.shape; + if (retyped(i)) blocky = false; + if (ax >= osh.size()) blocky = false; + for (size_t d = 0; blocky && d < ax; ++d) if (osh[d] != 1) blocky = false; + long fi = idxof(nd.input_ids[0]); + if (blocky && fi >= 0) { + const auto& ish = nodes[(size_t)fi]->output_buffer.shape; + size_t ip = 1, op2 = 1; + for (size_t d : ish) ip *= d; + for (size_t d : osh) op2 *= d; + if (ip * cnt != op2) blocky = false; + } else blocky = false; + } + if (blocky) { + long m0 = -1; + { + long t = up_in(i, 0); + while (t >= 0 && nodes[(size_t)t]->op_type == OpType::PRECISION_CAST + && !nodes[(size_t)t]->input_ids.empty()) + t = up(idxof(nodes[(size_t)t]->input_ids[0])); + m0 = t; + } + if (m0 >= 0 && nodes[(size_t)m0]->op_type == OpType::MATMUL + && nodes[(size_t)m0]->output_buffer.shape.size() == 2 + && nodes[(size_t)m0]->output_buffer.precision == Precision::FP16) { + size_t M = nodes[(size_t)m0]->output_buffer.shape[0]; + size_t N = nodes[(size_t)m0]->output_buffer.shape[1]; + long ka = idxof(nodes[(size_t)m0]->input_ids[0]); + size_t K = ka >= 0 ? nodes[(size_t)ka]->output_buffer.shape.back() : 0; + bool f32out = nd.output_buffer.precision == Precision::FP32; + long parentA = -1, parentB = -1, offA0 = 0, offB0 = 0; + bool ok = K > 0; + bool f32a = false; + std::vector cover; + for (size_t j = 0; j < cnt && ok; ++j) { + long t = idxof(nd.input_ids[j]); + std::vector chain; + while (t >= 0 && (passthrough(*nodes[(size_t)t]) + || (nodes[(size_t)t]->op_type == OpType::PRECISION_CAST))) { + if (cons[(size_t)t].size() != 1) { t = -1; break; } + chain.push_back((size_t)t); + t = idxof(nodes[(size_t)t]->input_ids[0]); + } + if (t < 0 || nodes[(size_t)t]->op_type != OpType::MATMUL + || plan->action[(size_t)t] != -1 + || cons[(size_t)t].size() != 1 + || nodes[(size_t)t]->output_buffer.shape.size() != 2 + || nodes[(size_t)t]->output_buffer.shape[0] != M + || nodes[(size_t)t]->output_buffer.shape[1] != N + || nodes[(size_t)t]->params.pretransposed_rhs) { ok = false; break; } + long offA, offB; + long lin = idxof(nodes[(size_t)t]->input_ids[0]); + bool jf32 = false; + long lcast = -1; + if (lin >= 0 && nodes[(size_t)lin]->op_type == OpType::PRECISION_CAST + && nodes[(size_t)lin]->output_buffer.precision == Precision::FP16 + && cons[(size_t)lin].size() == 1 + && !nodes[(size_t)lin]->input_ids.empty()) { + long ci = idxof(nodes[(size_t)lin]->input_ids[0]); + if (ci >= 0 && nodes[(size_t)ci]->output_buffer.precision == Precision::FP32) { + jf32 = true; lcast = lin; lin = ci; + } + } + long pa = alias_base(lin, &offA); + long pb = alias_base(idxof(nodes[(size_t)t]->input_ids[1]), &offB); + Precision wantA = jf32 ? Precision::FP32 : Precision::FP16; + if (pa < 0 || pb < 0 + || nodes[(size_t)pa]->output_buffer.precision != wantA + || (jf32 && retyped((size_t)pa)) + || nodes[(size_t)pb]->output_buffer.precision != Precision::FP16) { ok = false; break; } + if (j == 0) { parentA = pa; parentB = pb; offA0 = offA; offB0 = offB; f32a = jf32; } + else if (jf32 != f32a) { ok = false; break; } + else if (pa != parentA || pb != parentB + || offA != offA0 + (long)(j * M * K) + || offB != offB0 + (long)(j * K * N)) { ok = false; break; } + cover.push_back((size_t)t); + if (lcast >= 0) cover.push_back((size_t)lcast); + for (size_t cnode : chain) cover.push_back(cnode); + } + if (ok) { + MetalCluster c; + c.rule = 8; + c.a0 = (size_t)parentA; c.a1 = (size_t)parentB; + c.a2 = (size_t)offA0; c.a3 = (size_t)offB0; + c.u0 = (uint32_t)cnt; c.u1 = (uint32_t)M; + c.b0 = K; c.b1 = N; c.b2 = f32out ? 1 : 0; + c.b3 = f32a ? 1 : 0; + c.b4 = i; + add_cluster(c, i, cover); + continue; + } + } + } + bool samesz = true; + { + long fi = idxof(nd.input_ids[0]); + if (fi < 0) samesz = false; + else { + size_t ip = nodes[(size_t)fi]->output_buffer.total_size; + samesz = ip * cnt == nd.output_buffer.total_size; + } + } + long tp0 = samesz ? up_in(i, 0) : -1; + if (tp0 >= 0 && nodes[(size_t)tp0]->op_type == OpType::TRANSPOSE + && nodes[(size_t)tp0]->output_buffer.precision == Precision::FP16 + && nd.output_buffer.precision == Precision::FP16 + && nodes[(size_t)tp0]->output_buffer.shape.size() + 1 <= 8 + && !is_noop_transpose(*nodes[(size_t)tp0])) { + const auto& perm0 = nodes[(size_t)tp0]->params.permutation; + const auto& tsh0 = nodes[(size_t)tp0]->output_buffer.shape; + long parentP = -1, offP0 = 0, strideP = -1; + size_t blk = nodes[(size_t)tp0]->output_buffer.total_size; + bool ok = !perm0.empty(); + std::vector cover; + for (size_t j = 0; j < cnt && ok; ++j) { + long t = idxof(nd.input_ids[j]); + std::vector chain; + while (t >= 0 && passthrough(*nodes[(size_t)t])) { + if (cons[(size_t)t].size() != 1) { t = -1; break; } + chain.push_back((size_t)t); + t = idxof(nodes[(size_t)t]->input_ids[0]); + } + if (t < 0 || nodes[(size_t)t]->op_type != OpType::TRANSPOSE + || plan->action[(size_t)t] != -1 + || cons[(size_t)t].size() != 1 + || nodes[(size_t)t]->params.permutation != perm0 + || nodes[(size_t)t]->output_buffer.shape != tsh0 + || nodes[(size_t)t]->output_buffer.precision != Precision::FP16) { ok = false; break; } + long offP; + long pp = alias_base(idxof(nodes[(size_t)t]->input_ids[0]), &offP); + if (pp < 0 || nodes[(size_t)pp]->output_buffer.precision != Precision::FP16) { ok = false; break; } + long ii = idxof(nodes[(size_t)t]->input_ids[0]); + if (ii < 0 || nodes[(size_t)ii]->output_buffer.total_size != blk) { ok = false; break; } + if (j == 0) { parentP = pp; offP0 = offP; } + else if (j == 1 && pp == parentP && offP > offP0) { strideP = offP - offP0; } + else if (j == 1 || pp != parentP || offP != offP0 + (long)j * strideP) { ok = false; break; } + cover.push_back((size_t)t); + for (size_t cnode : chain) cover.push_back(cnode); + } + if (ok && (strideP < 0 || cnt < 2)) ok = false; + if (ok) { + size_t ax = (size_t)nd.params.axis; + long fi = idxof(nd.input_ids[0]); + const auto& fsh = nodes[(size_t)fi]->output_buffer.shape; + if (ax >= fsh.size()) ok = false; + else { + size_t pre = 1; + for (size_t d = 0; d < ax && d < fsh.size(); ++d) pre *= fsh[d]; + size_t fpre = 1; + const auto& csh = nd.output_buffer.shape; + for (size_t d = 0; d < ax && d < csh.size(); ++d) fpre *= csh[d]; + if (pre != fpre) ok = false; + } + } + if (ok) { + MetalCluster c; + c.rule = 10; + c.a0 = (size_t)parentP; c.a1 = (size_t)up_in(i, 0); + c.a2 = (size_t)offP0; + c.u0 = (uint32_t)cnt; + c.u1 = (uint32_t)nd.params.axis; + c.b0 = (size_t)strideP; + c.b4 = i; + add_cluster(c, i, cover); + continue; + } + } + + long c0 = up_in(i, 0); + if (!blocky) c0 = -1; + if (c0 >= 0 && nodes[(size_t)c0]->op_type == OpType::CONV1D + && nodes[(size_t)c0]->output_buffer.precision == Precision::FP16) { + const auto& osh = nodes[(size_t)c0]->output_buffer.shape; + size_t Lout = osh.empty() ? 0 : osh.back(); + size_t stride = nodes[(size_t)c0]->params.stride ? nodes[(size_t)c0]->params.stride : 1; + long parentX = -1, parentW = -1, offX0 = 0, offW0 = 0; + size_t L = 0, Kk = 0; + bool ok = Lout > 0; + std::vector cover; + for (size_t j = 0; j < cnt && ok; ++j) { + long t = up(idxof(nd.input_ids[j])); + if (t < 0 || nodes[(size_t)t]->op_type != OpType::CONV1D + || plan->action[(size_t)t] != -1 + || cons[(size_t)t].size() != 1 + || nodes[(size_t)t]->input_ids.size() != 2 + || nodes[(size_t)t]->output_buffer.shape.back() != Lout) { ok = false; break; } + long offX, offW; + long px = alias_base(idxof(nodes[(size_t)t]->input_ids[0]), &offX); + long pw = alias_base(idxof(nodes[(size_t)t]->input_ids[1]), &offW); + if (px < 0 || pw < 0 + || nodes[(size_t)px]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)pw]->output_buffer.precision != Precision::FP16) { ok = false; break; } + long xin = idxof(nodes[(size_t)t]->input_ids[0]); + long win = idxof(nodes[(size_t)t]->input_ids[1]); + size_t Lj = nodes[(size_t)xin]->output_buffer.shape.back(); + size_t Kj = nodes[(size_t)win]->output_buffer.shape.back(); + if (j == 0) { + parentX = px; parentW = pw; offX0 = offX; offW0 = offW; L = Lj; Kk = Kj; + if ((L - Kk) / stride + 1 != Lout) { ok = false; break; } + } else if (px != parentX || pw != parentW || Lj != L || Kj != Kk + || offX != offX0 + (long)(j * L) + || offW != offW0 + (long)(j * Kk)) { ok = false; break; } + cover.push_back((size_t)t); + } + if (ok) { + MetalCluster c; + c.rule = 9; + c.a0 = (size_t)parentX; c.a1 = (size_t)parentW; + c.a2 = (size_t)offX0; c.a3 = (size_t)offW0; + c.u0 = (uint32_t)cnt; c.u1 = (uint32_t)L; + c.b0 = Lout; c.b1 = Kk; c.b2 = stride; + c.b4 = i; + add_cluster(c, i, cover); + continue; + } + } + } + + if (nd.op_type == OpType::ADD_CLIPPED && nd.input_ids.size() == 2) { + long n0 = up_in(i, 0), n1 = up_in(i, 1); + auto rms2_side = [&](long nrm, long* src, long* w) -> bool { + if (nrm < 0 || nodes[(size_t)nrm]->op_type != OpType::RMS_NORM + || nodes[(size_t)nrm]->input_ids.size() < 2 + || plan->action[(size_t)nrm] != -1 || pinned[(size_t)nrm] || retyped((size_t)nrm) + || nodes[(size_t)nrm]->output_buffer.precision != Precision::FP16 + || !sole_use(nrm, {i})) return false; + long s0 = up_in((size_t)nrm, 0); + long w0 = up_in((size_t)nrm, 1); + if (w0 >= 0) w0 = deep_f16_source(w0); + if (s0 < 0 || w0 < 0 + || nodes[(size_t)s0]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)w0]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)s0]->output_buffer.shape.empty() + || nodes[(size_t)s0]->output_buffer.total_size != + nodes[(size_t)s0]->output_buffer.shape.back()) return false; + *src = s0; *w = w0; + return true; + }; + size_t D2 = nd.output_buffer.shape.empty() ? 0 : nd.output_buffer.shape.back(); + long srcA = -1, wA = -1, srcB = -1, wB = -1; + if (n0 != n1 && D2 > 0 && nd.output_buffer.total_size == D2 + && nd.output_buffer.precision == Precision::FP16 && !retyped(i) + && rms2_side(n0, &srcA, &wA) && rms2_side(n1, &srcB, &wB) + && nodes[(size_t)srcA]->output_buffer.total_size == D2 + && nodes[(size_t)srcB]->output_buffer.total_size == D2 + && nodes[(size_t)wA]->output_buffer.total_size == D2 + && nodes[(size_t)wB]->output_buffer.total_size == D2) { + MetalCluster c; + c.rule = 18; + c.a0 = (size_t)srcA; c.a1 = (size_t)wA; + c.a2 = (size_t)srcB; c.a3 = (size_t)wB; + c.u1 = (uint32_t)D2; + c.f0 = nodes[(size_t)n0]->params.epsilon; + c.f1 = nodes[(size_t)n1]->params.epsilon; + c.b4 = i; + std::vector cover{(size_t)n0, (size_t)n1}; + if (add_cluster(c, i, cover)) continue; + } + long norm = -1, res = -1; + if (n0 >= 0 && nodes[(size_t)n0]->op_type == OpType::RMS_NORM) { norm = n0; res = n1; } + else if (n1 >= 0 && nodes[(size_t)n1]->op_type == OpType::RMS_NORM) { norm = n1; res = n0; } + if (norm >= 0 && res >= 0 && sole_use(norm, {i})) { + long src = up_in((size_t)norm, 0); + long w = up_in((size_t)norm, 1); + auto f16 = [&](long j) { + return j >= 0 && nodes[(size_t)j]->output_buffer.precision == Precision::FP16; + }; + if (w >= 0) w = deep_f16_source(w); + bool one_row = src >= 0 && !nodes[(size_t)src]->output_buffer.shape.empty() + && nodes[(size_t)src]->output_buffer.total_size == + nodes[(size_t)src]->output_buffer.shape.back(); + if (src >= 0 && w >= 0 && one_row && f16((long)i) && f16(src) && f16(res) && f16(w) && f16(norm)) { + MetalCluster c; + c.rule = 3; + c.a0 = (size_t)src; c.a1 = (size_t)w; c.a2 = (size_t)res; + c.b4 = i; + c.f0 = nodes[(size_t)norm]->params.epsilon; + std::vector cover; + cover.push_back((size_t)norm); + size_t h_node = i; + if (cons[i].size() == 1) { + long mt = (long)cons[i][0]; + while (mt >= 0 && passthrough(*nodes[(size_t)mt]) && cons[(size_t)mt].size() == 1) + mt = (long)cons[(size_t)mt][0]; + if (mt >= 0 && nodes[(size_t)mt]->op_type == OpType::MULTIPLY + && nodes[(size_t)mt]->input_ids.size() == 2 + && nodes[(size_t)mt]->output_buffer.precision == Precision::FP16) { + long o0 = up_in((size_t)mt, 0), o1 = up_in((size_t)mt, 1); + long scn = (o0 == (long)i || (o0 >= 0 && up(o0) == (long)i)) ? o1 : o0; + long base = scn == o1 ? o0 : o1; + if (base >= 0 && scn >= 0 + && nodes[(size_t)scn]->op_type == OpType::INPUT + && nodes[(size_t)scn]->output_buffer.total_size == 1 + && nodes[(size_t)scn]->output_buffer.precision == Precision::FP16) { + c.b1 = (size_t)scn; + c.u1 = 1u; + collect_chain(cons[i][0], (long)i, cover); + cover.push_back((size_t)mt); + h_node = (size_t)mt; + c.b4 = h_node; + cover.push_back(i); + } + } + } + long next_norm = -1; + for (size_t cc : cons[h_node]) { + long t = (long)cc; + while (t >= 0 && passthrough(*nodes[(size_t)t])) { + if (cons[(size_t)t].size() != 1) { t = -1; break; } + t = (long)cons[(size_t)t][0]; + } + if (t >= 0 && nodes[(size_t)t]->op_type == OpType::RMS_NORM + && up_in((size_t)t, 0) == (long)h_node + && nodes[(size_t)t]->output_buffer.precision == Precision::FP16) { + next_norm = t; + break; + } + } + if (next_norm >= 0) { + long nw = deep_f16_source(up_in((size_t)next_norm, 1)); + size_t ndim = nodes[(size_t)next_norm]->output_buffer.shape.empty() ? 0 + : nodes[(size_t)next_norm]->output_buffer.shape.back(); + size_t adim = nodes[i]->output_buffer.shape.empty() ? 0 + : nodes[i]->output_buffer.shape.back(); + if (nw >= 0 && ndim == adim && f16(nw)) { + c.u0 = 1u; + c.a4 = (size_t)nw; + c.a5 = (size_t)next_norm; + c.s1 = cactus_metal_alloc_shared(nodes[(size_t)next_norm]->output_buffer.total_size * 2); + cover.push_back((size_t)next_norm); + long ch = idxof(nodes[(size_t)next_norm]->input_ids[0]); + collect_chain((size_t)ch, (long)h_node, cover); + } + } + add_cluster(c, h_node, cover); + continue; + } + } + } + } + { + std::unordered_map> groups; + std::vector order; + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + const GraphNode& mn = *nodes[i]; + if (mn.op_type != OpType::MATMUL || mn.input_ids.size() < 2) continue; + long r = idxof(mn.input_ids[1]); + if (r < 0) continue; + const BufferDesc& rb = nodes[(size_t)r]->output_buffer; + size_t M = 1; + for (size_t d = 0; d + 1 < mn.output_buffer.shape.size(); ++d) M *= mn.output_buffer.shape[d]; + if (M != 1 || !PrecisionTraits::is_cq(rb.precision) || rb.group_size != 128 + || PrecisionTraits::cq_bits(rb.precision) != 4 + || (rb.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) continue; + long src = up_in(i, 0); + if (src < 0) continue; + auto it = groups.find(src); + if (it == groups.end()) { groups[src] = {i}; order.push_back(src); } + else it->second.push_back(i); + } + for (long src : order) { + auto& mms = groups[src]; + if (mms.size() < 2) continue; + for (size_t base = 0; base + 1 < mms.size(); base += 3) { + size_t cnt = std::min(3, mms.size() - base); + if (cnt < 2) break; + size_t anchor_i = mms[base]; + MetalCluster c; + c.rule = 2; + c.a0 = (size_t)src; + c.b0 = mms[base]; + c.b1 = mms[base + 1]; + c.b2 = cnt > 2 ? mms[base + 2] : 0; + c.u0 = (uint32_t)cnt; + c.b4 = anchor_i; + size_t maxK = 0; + for (size_t mi = 0; mi < cnt; ++mi) { + const BufferDesc& rb = nodes[(size_t)idxof(nodes[mms[base + mi]]->input_ids[1])]->output_buffer; + CactusQuantMatrix W = rb.to_cq_matrix(); + if (W.K > maxK) maxK = W.K; + } + for (uint32_t bi = 0; bi < c.u0; ++bi) + if (!c.sc[bi]) c.sc[bi] = cactus_metal_alloc_shared(maxK * 2); + std::vector cover; + if (add_cluster(c, anchor_i, cover)) + for (size_t mi = 1; mi < cnt; ++mi) plan->action[mms[base + mi]] = -3; + } + } + } + + if (!cands.empty()) { + std::vector mark(n, 0); + for (auto& cd : cands) { + mark[cd.anchor] = 2; + for (size_t v : cd.cover) if (mark[v] == 0) mark[v] = 1; + } + for (size_t ri = n; ri-- > 0;) { + if (mark[ri]) continue; + const GraphNode& ndp = *nodes[ri]; + if (!(is_alias_op(ndp.op_type) || is_same_cast(ndp, nodes, map) || is_noop_transpose(ndp))) continue; + if (cons[ri].empty()) { mark[ri] = 3; continue; } + bool all_in = true; + for (size_t cc : cons[ri]) if (!mark[cc]) { all_in = false; break; } + if (all_in) mark[ri] = 3; + } + bool valid = true; + for (size_t v = 0; v < n && valid; ++v) { + if (mark[v] != 1 && mark[v] != 3) continue; + for (size_t cc : cons[v]) { + if (!mark[cc]) { + valid = false; + break; + } + } + } + for (size_t v = 0; v < n && valid; ++v) + if ((mark[v] == 1 || mark[v] == 3) && pinned[v]) valid = false; + if (valid) { + for (size_t v = 0; v < n; ++v) + if ((mark[v] == 1 || mark[v] == 3) && plan->action[v] == -1) plan->action[v] = -2; + for (auto& cd : cands) { + int32_t cid = (int32_t)plan->clusters.size(); + plan->clusters.push_back(cd.c); + plan->action[cd.anchor] = cid; + } + } + } + + for (auto& cl : plan->clusters) { + if (cl.rule != 5) continue; + const auto& sh0 = nodes[0]->output_buffer; + const auto& sh1 = n > 1 ? nodes[1]->output_buffer : nodes[0]->output_buffer; + const auto& sh2 = n > 2 ? nodes[2]->output_buffer : nodes[0]->output_buffer; + if (n > 2 && nodes[0]->op_type == OpType::INPUT && nodes[1]->op_type == OpType::INPUT + && nodes[2]->op_type == OpType::INPUT + && sh0.precision == Precision::FP16 && sh0.total_size > 0 + && sh1.precision == Precision::FP16 && sh1.shape.size() >= 2 + && sh2.total_size == 1) { + plan->fold_h = 0; + plan->fold_ple = 1; + plan->fold_pos = 2; + plan->fold_w = (long)cl.a1; + plan->fold_nl = sh1.shape[sh1.shape.size() - 2]; + plan->fold_pd = sh1.shape.back(); + } + break; + } + + { + auto ew_kind = [&](const GraphNode& nd, int* kind, int* code, float* p0, float* p1) -> bool { + *p0 = 0; *p1 = 0; + switch (nd.op_type) { + case OpType::GELU: *kind = 0; *code = 0; return true; + case OpType::TANH: *kind = 0; *code = 1; return true; + case OpType::SILU: *kind = 0; *code = 2; return true; + case OpType::RELU: *kind = 0; *code = 3; return true; + case OpType::GELU_ERF: *kind = 0; *code = 4; return true; + case OpType::SIGMOID: *kind = 0; *code = 5; return true; + case OpType::SCALAR_ADD: *kind = 1; *code = 0; *p0 = nd.params.scalar; return true; + case OpType::SCALAR_SUBTRACT: *kind = 1; *code = 1; *p0 = nd.params.scalar; return true; + case OpType::SCALAR_MULTIPLY: *kind = 1; *code = 2; *p0 = nd.params.scalar; return true; + case OpType::SCALAR_DIVIDE: *kind = 1; *code = 3; *p0 = nd.params.scalar; return true; + case OpType::SCALAR_EXP: *kind = 1; *code = 4; return true; + case OpType::SCALAR_SQRT: *kind = 1; *code = 5; return true; + case OpType::SCALAR_COS: *kind = 1; *code = 6; return true; + case OpType::SCALAR_SIN: *kind = 1; *code = 7; return true; + case OpType::SCALAR_LOG: *kind = 1; *code = 8; return true; + case OpType::ABS: *kind = 1; *code = 9; return true; + case OpType::POW: *kind = 1; *code = 10; *p0 = nd.params.scalar; return true; + case OpType::SCALAR_NOT_EQUAL: *kind = 1; *code = 11; *p0 = nd.params.scalar; return true; + case OpType::LEAKY_RELU: *kind = 1; *code = 12; *p0 = nd.params.scalar; return true; + case OpType::ADD: *kind = 2; *code = 0; return true; + case OpType::ADD_CLIPPED: *kind = 2; *code = 1; return true; + case OpType::SUBTRACT: *kind = 2; *code = 2; return true; + case OpType::MULTIPLY: *kind = 2; *code = 3; return true; + case OpType::DIVIDE: *kind = 2; *code = 4; return true; + case OpType::NOT_EQUAL: *kind = 2; *code = 5; return true; + case OpType::CLAMP: *kind = 3; *code = 0; *p0 = nd.params.scalar; *p1 = nd.params.scale; return true; + case OpType::PRECISION_CAST: *kind = 4; + *code = nd.output_buffer.precision == Precision::FP32 ? 1 : 0; + return nd.output_buffer.precision == Precision::FP16 + || nd.output_buffer.precision == Precision::FP32; + default: return false; + } + }; + auto prec_ok = [](Precision p) { return p == Precision::FP16 || p == Precision::FP32; }; + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd0 = *nodes[i]; + if (nd0.op_type != OpType::RMS_NORM || nd0.input_ids.size() < 2) continue; + if (nd0.output_buffer.precision != Precision::FP16 || retyped(i)) continue; + const auto& osh = nd0.output_buffer.shape; + if (osh.empty()) continue; + size_t dim = osh.back(); + size_t rows = nd0.output_buffer.total_size / dim; + if (rows < 2 || dim < 32) continue; + long ai = idxof(nd0.input_ids[0]); + long wi = idxof(nd0.input_ids[1]); + if (ai < 0 || wi < 0 || plan->action[(size_t)ai] != -1 || retyped((size_t)ai)) continue; + const GraphNode& an = *nodes[(size_t)ai]; + if ((an.op_type != OpType::ADD && an.op_type != OpType::ADD_CLIPPED) + || an.input_ids.size() != 2 + || an.output_buffer.precision != Precision::FP16 + || an.output_buffer.total_size != nd0.output_buffer.total_size + || pinned[(size_t)ai]) continue; + long x0 = idxof(an.input_ids[0]); + long x1 = idxof(an.input_ids[1]); + if (x0 < 0 || x1 < 0 + || nodes[(size_t)x0]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)x1]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)x0]->output_buffer.total_size != an.output_buffer.total_size + || nodes[(size_t)x1]->output_buffer.total_size != an.output_buffer.total_size) continue; + long ww = deep_f16_source(wi); + if (ww < 0 || nodes[(size_t)ww]->output_buffer.total_size != dim) continue; + bool ordered = true; + for (size_t cc : cons[(size_t)ai]) + if (cc < i) { ordered = false; break; } + if (!ordered) continue; + MetalCluster c; + c.rule = 12; + c.a0 = (size_t)x0; c.a1 = (size_t)x1; c.a2 = (size_t)ai; c.a3 = (size_t)ww; + c.u0 = (uint32_t)rows; c.u1 = (uint32_t)dim; + c.f0 = nd0.params.epsilon; + c.b0 = an.op_type == OpType::ADD_CLIPPED ? 1 : 0; + c.b4 = i; + std::vector cover; + if (add_cluster(c, i, cover)) plan->action[(size_t)ai] = -3; + } + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd0 = *nodes[i]; + if (nd0.op_type != OpType::ADD || nd0.input_ids.size() != 2) continue; + if (nd0.output_buffer.precision != Precision::FP16 || retyped(i)) continue; + long i0 = idxof(nd0.input_ids[0]); + long i1 = idxof(nd0.input_ids[1]); + if (i0 < 0 || i1 < 0 || i0 == i1) continue; + long mm = -1, bs = -1; + if (nodes[(size_t)i0]->op_type == OpType::MATMUL) { mm = i0; bs = i1; } + else if (nodes[(size_t)i1]->op_type == OpType::MATMUL) { mm = i1; bs = i0; } + if (mm < 0 || plan->action[(size_t)mm] != -1 || pinned[(size_t)mm]) continue; + if (plan->action[(size_t)bs] == -2) continue; + if (retyped((size_t)mm) || retyped((size_t)bs)) continue; + if (cons[(size_t)mm].size() != 1) continue; + GraphNode& m = *nodes[(size_t)mm]; + if (m.input_ids.size() < 2) continue; + long li = idxof(m.input_ids[0]); + long ri = idxof(m.input_ids[1]); + if (li < 0 || ri < 0 || retyped((size_t)li) || retyped((size_t)ri)) continue; + if (plan->action[(size_t)li] == -2 || plan->action[(size_t)ri] == -2) continue; + const BufferDesc& lb = nodes[(size_t)li]->output_buffer; + const BufferDesc& rb = nodes[(size_t)ri]->output_buffer; + const BufferDesc& bb = nodes[(size_t)bs]->output_buffer; + if (lb.precision != Precision::FP16 || rb.precision != Precision::FP16 + || bb.precision != Precision::FP16) continue; + if (rb.shape.size() != 2 || lb.shape.empty()) continue; + size_t K = lb.shape.back(); + if (K == 0 || lb.total_size != K) continue; + size_t N = m.params.pretransposed_rhs ? rb.shape[0] : rb.shape[1]; + size_t rk = m.params.pretransposed_rhs ? rb.shape[1] : rb.shape[0]; + if (rk != K || N == 0 || m.output_buffer.total_size != N) continue; + if (bb.total_size != N || nd0.output_buffer.total_size != N) continue; + MetalCluster c; + c.rule = 13; + c.a0 = (size_t)li; c.a1 = (size_t)ri; c.a2 = (size_t)bs; + c.u0 = (uint32_t)K; c.u1 = (uint32_t)N; + c.b0 = m.params.pretransposed_rhs ? 1 : 0; + c.b4 = i; + std::vector cover; + cover.push_back((size_t)mm); + add_cluster(c, i, cover); + } + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd0 = *nodes[i]; + if (nd0.op_type != OpType::ADD || nd0.input_ids.size() != 2) continue; + if (nd0.output_buffer.precision != Precision::FP16 || retyped(i)) continue; + const auto& osh = nd0.output_buffer.shape; + if (osh.size() < 2) continue; + size_t D = osh.back(); + if (D == 0 || (D % 2) != 0) continue; + size_t H = nd0.output_buffer.total_size / D; + if (H == 0 || H * D != nd0.output_buffer.total_size) continue; + long mA = up_in(i, 0), mB = up_in(i, 1); + if (mA < 0 || mB < 0 || mA == mB) continue; + auto is_mul = [&](long j) { + return nodes[(size_t)j]->op_type == OpType::MULTIPLY + && nodes[(size_t)j]->input_ids.size() == 2 + && plan->action[(size_t)j] == -1 && !pinned[(size_t)j] && !retyped((size_t)j) + && nodes[(size_t)j]->output_buffer.precision == Precision::FP16 + && nodes[(size_t)j]->output_buffer.total_size == H * D; + }; + if (!is_mul(mA) || !is_mul(mB)) continue; + auto split_mul = [&](long m, long* big, long* small) -> bool { + long i0 = up_in((size_t)m, 0), i1 = up_in((size_t)m, 1); + if (i0 < 0 || i1 < 0) return false; + size_t t0 = nodes[(size_t)i0]->output_buffer.total_size; + size_t t1 = nodes[(size_t)i1]->output_buffer.total_size; + if (t0 == H * D && t1 == D) { *big = i0; *small = i1; return true; } + if (t1 == H * D && t0 == D) { *big = i1; *small = i0; return true; } + return false; + }; + long xA = -1, cA = -1, xB = -1, sB = -1; + if (!split_mul(mA, &xA, &cA) || !split_mul(mB, &xB, &sB)) continue; + long mulCos = -1, mulSin = -1, xi = -1, ci = -1, si = -1, cat = -1; + if (nodes[(size_t)xB]->op_type == OpType::CAT) { + mulCos = mA; mulSin = mB; xi = xA; ci = cA; si = sB; cat = xB; + } else if (nodes[(size_t)xA]->op_type == OpType::CAT) { + mulCos = mB; mulSin = mA; xi = xB; ci = sB; si = cA; cat = xA; + } else continue; + GraphNode& cn = *nodes[(size_t)cat]; + if (cn.input_ids.size() != 2 || plan->action[(size_t)cat] != -1 + || pinned[(size_t)cat] || retyped((size_t)cat)) continue; + if (cn.output_buffer.total_size != H * D) continue; + size_t cat_ax = (size_t)cn.params.axis; + if (cat_ax != cn.output_buffer.shape.size() - 1) continue; + long neg = up_in((size_t)cat, 0); + long lo = up_in((size_t)cat, 1); + if (neg < 0 || lo < 0) continue; + GraphNode& ng = *nodes[(size_t)neg]; + if (ng.op_type != OpType::SCALAR_MULTIPLY || ng.params.scalar != -1.0f + || plan->action[(size_t)neg] != -1 || pinned[(size_t)neg] || retyped((size_t)neg)) continue; + long hi = up_in((size_t)neg, 0); + if (hi < 0) continue; + GraphNode& hs = *nodes[(size_t)hi]; + GraphNode& ls = *nodes[(size_t)lo]; + if (hs.op_type != OpType::SLICE || ls.op_type != OpType::SLICE) continue; + if (plan->action[(size_t)hi] != -1 || plan->action[(size_t)lo] != -1 + || pinned[(size_t)hi] || pinned[(size_t)lo]) continue; + if (hs.params.slice_start != D / 2 || ls.params.slice_start != 0) continue; + size_t hlen = hs.params.slice_length ? hs.params.slice_length : D / 2; + size_t llen = ls.params.slice_length ? ls.params.slice_length : D / 2; + if (hlen != D / 2 || llen != D / 2) continue; + if ((size_t)hs.params.axis != hs.output_buffer.shape.size() - 1 + || (size_t)ls.params.axis != ls.output_buffer.shape.size() - 1) continue; + long xh = up_in((size_t)hi, 0), xl = up_in((size_t)lo, 0); + if (xh != xi || xl != xi) continue; + if (nodes[(size_t)xi]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)ci]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)si]->output_buffer.precision != Precision::FP16) continue; + if (!sole_use(mulCos, {i}) || !sole_use(mulSin, {i}) + || !sole_use(cat, {(size_t)mulSin}) || !sole_use(neg, {(size_t)cat}) + || !sole_use(hi, {(size_t)neg}) || !sole_use(lo, {(size_t)cat})) continue; + MetalCluster c; + c.rule = 14; + c.a0 = (size_t)xi; c.a1 = (size_t)ci; c.a2 = (size_t)si; + c.u0 = (uint32_t)H; c.u1 = (uint32_t)D; + c.b4 = i; + std::vector cover{(size_t)mulCos, (size_t)mulSin, (size_t)cat, + (size_t)neg, (size_t)hi, (size_t)lo}; + collect_chain((size_t)idxof(nd0.input_ids[0]), mulCos, cover); + collect_chain((size_t)idxof(nd0.input_ids[1]), mulSin, cover); + GraphNode& xn = *nodes[(size_t)xi]; + if (D <= 1024 && xn.op_type == OpType::RMS_NORM && xn.input_ids.size() >= 2 + && plan->action[(size_t)xi] == -1 && !pinned[(size_t)xi] && !retyped((size_t)xi) + && sole_use(xi, {(size_t)mulCos, (size_t)hi, (size_t)lo})) { + long rsrc = up_in((size_t)xi, 0); + long rw = up_in((size_t)xi, 1); + if (rw >= 0) rw = deep_f16_source(rw); + if (rsrc >= 0 && rw >= 0 + && nodes[(size_t)rsrc]->output_buffer.precision == Precision::FP16 + && nodes[(size_t)rsrc]->output_buffer.total_size == H * D + && nodes[(size_t)rw]->output_buffer.precision == Precision::FP16 + && nodes[(size_t)rw]->output_buffer.total_size == D) { + c.rule = 17; + c.a0 = (size_t)rsrc; c.a3 = (size_t)rw; + c.f0 = xn.params.epsilon; + cover.push_back((size_t)xi); + } + } + add_cluster(c, i, cover); + } + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd0 = *nodes[i]; + if (nd0.op_type != OpType::SCALAR_MULTIPLY || nd0.input_ids.empty()) continue; + if (nd0.output_buffer.precision != Precision::FP16 || retyped(i)) continue; + const auto& osh = nd0.output_buffer.shape; + if (osh.empty()) continue; + size_t D = osh.back(); + size_t rows = nd0.output_buffer.total_size / D; + if (D < 32 || rows == 0 || rows * D != nd0.output_buffer.total_size) continue; + long rn = up_in(i, 0); + if (rn < 0 || nodes[(size_t)rn]->op_type != OpType::RMS_NORM + || nodes[(size_t)rn]->input_ids.size() < 2 + || plan->action[(size_t)rn] != -1 || pinned[(size_t)rn] || retyped((size_t)rn) + || nodes[(size_t)rn]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)rn]->output_buffer.total_size != rows * D) continue; + if (!sole_use(rn, {i})) continue; + long src = up_in((size_t)rn, 0); + long w = up_in((size_t)rn, 1); + if (w >= 0) w = deep_f16_source(w); + if (src < 0 || w < 0 + || nodes[(size_t)src]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)w]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)w]->output_buffer.total_size != D) continue; + MetalCluster c; + c.rule = 15; + c.a0 = (size_t)src; c.a1 = (size_t)w; + c.u0 = (uint32_t)rows; c.u1 = (uint32_t)D; + c.f0 = nodes[(size_t)rn]->params.epsilon; + c.f1 = nd0.params.scalar; + c.b4 = i; + std::vector cover{(size_t)rn}; + collect_chain((size_t)idxof(nd0.input_ids[0]), rn, cover); + add_cluster(c, i, cover); + } + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + GraphNode& nd0 = *nodes[i]; + if (nd0.op_type != OpType::TOPK || nd0.input_ids.empty()) continue; + if (nd0.output_buffer.precision != Precision::FP32) continue; + size_t k = nd0.params.top_k; + if (k == 0 || k > 16) continue; + long sm = up_in(i, 0); + if (sm < 0 || nodes[(size_t)sm]->op_type != OpType::SCALAR_MULTIPLY + || nodes[(size_t)sm]->input_ids.empty() + || plan->action[(size_t)sm] != -1 || pinned[(size_t)sm] || retyped((size_t)sm) + || nodes[(size_t)sm]->output_buffer.precision != Precision::FP16) continue; + const auto& ssh = nodes[(size_t)sm]->output_buffer.shape; + if (ssh.size() != 2) continue; + size_t rows = ssh[0], E = ssh[1]; + if (rows == 0 || E == 0 || E > 4096) continue; + if (nd0.output_buffer.total_size != 2 * rows * k) continue; + long sfm = -1; + for (size_t cc : cons[(size_t)sm]) { + if (cc == i) continue; + const GraphNode& cn = *nodes[cc]; + if (cn.op_type == OpType::SOFTMAX && plan->action[cc] == -1 + && !pinned[cc] && !retyped(cc) + && cn.output_buffer.precision == Precision::FP16 + && cn.output_buffer.total_size == rows * E) { + if (sfm < 0) sfm = (long)cc; + else { sfm = -1; break; } + } else if (!is_alias_op(cn.op_type)) { sfm = -1; break; } + } + if (sfm < 0 || !sole_use(sm, {i, (size_t)sfm})) continue; + if ((size_t)sfm > i) continue; + bool ordered = true; + for (size_t cc : cons[(size_t)sfm]) + if (cc < i) { ordered = false; break; } + if (!ordered) continue; + long lg = up_in((size_t)sm, 0); + if (lg < 0 || nodes[(size_t)lg]->output_buffer.precision != Precision::FP16 + || nodes[(size_t)lg]->output_buffer.total_size != rows * E) continue; + MetalCluster c; + c.rule = 16; + c.a0 = (size_t)lg; c.a1 = (size_t)sfm; + c.u0 = (uint32_t)rows; c.u1 = (uint32_t)E; + c.b0 = k; + c.f0 = nodes[(size_t)sm]->params.scalar; + c.b4 = i; + std::vector cover{(size_t)sm}; + collect_chain((size_t)idxof(nd0.input_ids[0]), sm, cover); + if (add_cluster(c, i, cover)) plan->action[(size_t)sfm] = -3; + } + for (size_t i = 0; i < n; ++i) { + if (plan->action[i] != -1) continue; + int kind, code; float p0, p1; + if (!ew_kind(*nodes[i], &kind, &code, &p0, &p1)) continue; + if (!prec_ok(nodes[i]->output_buffer.precision) || retyped(i)) continue; + size_t total = nodes[i]->output_buffer.total_size; + std::vector chain{i}; + size_t cur = i; + while (chain.size() < 12) { + if (cons[cur].size() != 1) break; + size_t nxt = cons[cur][0]; + if (plan->action[nxt] != -1) break; + int k2, c2; float q0, q1; + if (!ew_kind(*nodes[nxt], &k2, &c2, &q0, &q1)) break; + if (!prec_ok(nodes[nxt]->output_buffer.precision) || retyped(nxt)) break; + if (nodes[nxt]->output_buffer.total_size != total) break; + bool feeds = false; + for (size_t a = 0; a < nodes[nxt]->input_ids.size() && a < 2; ++a) + if (idxof(nodes[nxt]->input_ids[a]) == (long)cur) feeds = true; + if (!feeds) break; + chain.push_back(nxt); + cur = nxt; + } + size_t real_ops = 0; + for (size_t ci = 0; ci < chain.size(); ++ci) { + int k2, c2; float q0, q1; + ew_kind(*nodes[chain[ci]], &k2, &c2, &q0, &q1); + if (k2 != 4) ++real_ops; + } + if (chain.size() < 2 || real_ops == 0) continue; + std::vector blob; + std::vector sides; + std::vector side_f32; + bool ok = true; + long head_in = -1; + Precision run = Precision::FP16; + for (size_t ci = 0; ci < chain.size() && ok; ++ci) { + const GraphNode& cn = *nodes[chain[ci]]; + int k2, c2; float q0, q1; + ew_kind(cn, &k2, &c2, &q0, &q1); + long prev = ci == 0 ? -1 : (long)chain[ci - 1]; + if (ci == 0) { + long hi2 = idxof(cn.input_ids[0]); + if (hi2 < 0 || !prec_ok(nodes[(size_t)hi2]->output_buffer.precision) + || retyped((size_t)hi2) + || nodes[(size_t)hi2]->output_buffer.total_size != total) { ok = false; break; } + head_in = hi2; + run = nodes[(size_t)hi2]->output_buffer.precision; + } + if (k2 == 2) { + long in0 = idxof(cn.input_ids[0]); + long in1 = idxof(cn.input_ids[1]); + long side, chain_in; + bool rhs; + if (ci > 0 && in0 == prev) { chain_in = in0; side = in1; rhs = false; } + else if (ci > 0 && in1 == prev) { chain_in = in1; side = in0; rhs = true; } + else { chain_in = in0; side = in1; rhs = false; } + if (side < 0 || chain_in < 0 + || nodes[(size_t)side]->output_buffer.precision != run + || retyped((size_t)side) + || plan->action[(size_t)side] == -2) { ok = false; break; } + size_t inner = nodes[i]->output_buffer.shape.empty() + ? 1 : nodes[i]->output_buffer.shape.back(); + size_t stotal = nodes[(size_t)side]->output_buffer.total_size; + int bmode; + if (stotal == total) bmode = 0; + else if (inner > 1 && stotal * inner == total + && !nodes[(size_t)side]->output_buffer.shape.empty() + && nodes[(size_t)side]->output_buffer.shape.back() == 1) bmode = 1; + else if (inner > 1 && stotal == inner) bmode = 2; + else { ok = false; break; } + size_t slot = sides.size(); + for (size_t si = 0; si < sides.size(); ++si) + if (sides[si] == (size_t)side) slot = si; + if (slot == sides.size()) { + if (sides.size() >= 3) { ok = false; break; } + sides.push_back((size_t)side); + side_f32.push_back(run == Precision::FP32 ? 1 : 0); + } else if (side_f32[slot] != (run == Precision::FP32 ? 1 : 0)) { ok = false; break; } + c2 |= (rhs ? 16 : 0) | (run == Precision::FP32 ? 32 : 0) | ((int)slot << 6) | (bmode << 8); + } + if (k2 == 4) { + Precision from = ci == 0 ? run : nodes[(size_t)prev]->output_buffer.precision; + if (from == cn.output_buffer.precision) { ok = false; break; } + run = cn.output_buffer.precision; + } else if (cn.output_buffer.precision != run) { ok = false; break; } + EwChainStep st{k2, c2, q0, q1}; + float packed[4]; + std::memcpy(packed, &st, sizeof(st)); + blob.insert(blob.end(), packed, packed + 4); + } + if (!ok || head_in < 0) continue; + Precision head_prec = nodes[(size_t)head_in]->output_buffer.precision; + uint32_t flags = (head_prec == Precision::FP32 ? 1u : 0u) + | (nodes[chain.back()]->output_buffer.precision == Precision::FP32 ? 2u : 0u); + for (size_t si = 0; si < sides.size(); ++si) + if (side_f32[si]) flags |= (4u << si); + MetalCluster c; + c.rule = 11; + c.a0 = (size_t)head_in; + c.a1 = sides.size() > 0 ? sides[0] : (size_t)0; + c.a2 = sides.size() > 1 ? sides[1] : (size_t)0; + c.a3 = sides.size() > 2 ? sides[2] : (size_t)0; + c.u0 = (uint32_t)chain.size(); + c.u1 = (uint32_t)sides.size(); + c.b0 = plan->blobs.size(); + c.b1 = flags; + c.b4 = chain.back(); + plan->blobs.push_back(std::move(blob)); + std::vector cover(chain.begin(), chain.end() - 1); + add_cluster(c, chain.back(), cover); + } + } + + plan->exec_list.reserve(n); + for (size_t i = 0; i < n; ++i) + if (plan->action[i] != -2 && nodes[i]->op_type != OpType::INPUT) + plan->exec_list.push_back((uint32_t)i); + + bool decode_like = false; + for (auto& cl : plan->clusters) if (cl.rule == 1) { decode_like = true; break; } + if (decode_like) { + std::vector last_read(n, 0); + auto base_of = [&](size_t i) -> size_t { + size_t cur = i; + while (passthrough(*nodes[cur]) || plan->action[cur] == -2 || is_noop_transpose(*nodes[cur]) + || nodes[cur]->op_type == OpType::SLICE || nodes[cur]->op_type == OpType::INDEX) { + if (nodes[cur]->input_ids.empty()) break; + long j = idxof(nodes[cur]->input_ids[0]); + if (j < 0) break; + if (plan->action[cur] != -2 && !passthrough(*nodes[cur]) + && !is_noop_transpose(*nodes[cur])) { + bool aliasing = false; + const GraphNode& sn = *nodes[cur]; + if (sn.op_type == OpType::INDEX && sn.params.axis == 0) aliasing = true; + if (sn.op_type == OpType::SLICE) { + size_t ax = (size_t)sn.params.axis; + long jj = idxof(sn.input_ids[0]); + if (jj >= 0 && ax < nodes[(size_t)jj]->output_buffer.shape.size()) { + size_t outer = 1; + for (size_t d = 0; d < ax; ++d) outer *= nodes[(size_t)jj]->output_buffer.shape[d]; + if (outer == 1) aliasing = true; + } + } + if (!aliasing) break; + } + cur = (size_t)j; + } + return cur; + }; + for (size_t i = 0; i < n; ++i) { + for (size_t id : nodes[i]->input_ids) { + long j = idxof(id); + if (j >= 0) { + size_t b = base_of((size_t)j); + if (i > last_read[b]) last_read[b] = i; + } + } + } + for (auto& cl : plan->clusters) { + size_t anchor = cl.b4 ? cl.b4 : (cl.rule == 2 ? std::max(cl.b0, std::max(cl.b1, cl.b2)) : 0); + if (!anchor) continue; + for (size_t arg : {cl.a0, cl.a1, cl.a2, cl.a3, cl.a4, cl.a5, cl.b0, cl.b1, cl.b2, cl.b3}) { + if (arg == 0 || arg >= n) continue; + size_t b = base_of(arg); + if (anchor > last_read[b]) last_read[b] = anchor; + } + } + plan->arena_off.assign(n, -1); + struct Range { size_t off, size, free_at; }; + std::vector live; + size_t cursor = 0, peak = 0; + for (uint32_t ui : plan->exec_list) { + size_t i = ui; + const GraphNode& nd = *nodes[i]; + if (plan->action[i] == -2) continue; + if (pinned[i]) continue; + if (nd.op_type == OpType::KV_CACHE_STATE || nd.op_type == OpType::CONV_CACHE_STATE + || nd.op_type == OpType::RECURRENT_CACHE_STATE || nd.op_type == OpType::KV_CACHE_APPEND + || nd.op_type == OpType::PERSISTENT) continue; + if (passthrough(nd) || is_noop_transpose(nd)) continue; + if (nd.op_type == OpType::SLICE || nd.op_type == OpType::INDEX) continue; + if (last_read[i] == 0) continue; + size_t need = (nd.output_buffer.byte_size + 255) & ~size_t(255); + if (need == 0 || need > (8u << 20)) continue; + bool placed = false; + for (auto& r : live) { + if (r.free_at < i && r.size >= need) { + plan->arena_off[i] = (long)r.off; + r.free_at = last_read[i]; + placed = true; + break; + } + } + if (!placed) { + plan->arena_off[i] = (long)cursor; + live.push_back({cursor, need, last_read[i]}); + cursor += need; + if (cursor > peak) peak = cursor; + } + } + if (peak > 0 && peak < (256u << 20)) + plan->arena_base = (char*)cactus_metal_alloc_shared(peak); + if (!plan->arena_base) plan->arena_off.assign(n, -1); + } + + return plan; +} + +static void release_cluster_buffers(MetalFusePlan* p) { + for (auto& c : p->clusters) { + if (c.s0) { cactus_metal_free_shared(c.s0); c.s0 = nullptr; } + if (c.s1) { cactus_metal_free_shared(c.s1); c.s1 = nullptr; } + for (auto& s : c.sc) if (s) { cactus_metal_free_shared(s); s = nullptr; } + } +} + +void cactus_metal_plan_free(MetalFusePlan* p) { + if (!p) return; + release_cluster_buffers(p); + if (p->arena_base) { cactus_metal_free_shared(p->arena_base); p->arena_base = nullptr; } + delete p; +} + +bool cactus_metal_plan_has_arena(const MetalFusePlan* p) { + if (!p) return false; + if (p->arena_base) return true; + for (const auto& c : p->clusters) if (c.rule == 1) return true; + return false; +} + +void cactus_metal_plan_extend_last_use(const MetalFusePlan* p, std::vector& last_use) { + if (!p) return; + const size_t n = last_use.size(); + for (const auto& c : p->clusters) { + size_t ai = c.b4; + if (ai >= n) continue; + for (size_t v : {c.a0, c.a1, c.a2, c.a3, c.a4, c.a5, c.b0, c.b1, c.b2, c.b3}) + if (v < n && last_use[v] < ai) last_use[v] = ai; + } +} + +int32_t cactus_metal_plan_action(const MetalFusePlan* p, size_t i) { + return p && i < p->action.size() ? p->action[i] : -1; +} + + +const std::vector* cactus_metal_plan_exec_list(const MetalFusePlan* p) { + return p ? &p->exec_list : nullptr; +} + +void* cactus_metal_plan_arena_ptr(const MetalFusePlan* p, size_t i) { + if (!p || !p->arena_base || i >= p->arena_off.size() || p->arena_off[i] < 0) return nullptr; + return p->arena_base + p->arena_off[i]; +} + +bool cactus_metal_plan_fold(MetalFusePlan* p, + const std::vector>& nodes) { + if (!p || p->fold_h < 0 || !cactus_graph_fused_embed()) return false; + void* h = nodes[(size_t)p->fold_h]->output_buffer.get_data(); + void* ple = nodes[(size_t)p->fold_ple]->output_buffer.get_data(); + void* pos = nodes[(size_t)p->fold_pos]->output_buffer.get_data(); + if (!h || !ple) return false; + CactusQuantMatrix W = nodes[(size_t)p->fold_w]->output_buffer.to_cq_matrix(); + return cactus_graph_metal_fold_prologue(h, ple, pos, &W, p->fold_nl, p->fold_pd); +} + +bool cactus_metal_plan_encode(MetalFusePlan* p, int32_t cid, + const std::vector>& nodes, + const std::unordered_map& map) { + MetalCluster& c = p->clusters[(size_t)cid]; + auto data = [&](size_t i) { return nodes[i]->output_buffer.get_data(); }; + switch (c.rule) { + case 1: { + GraphNode& anchor = *nodes[c.b4]; + BufferDesc& kc = nodes[c.b2]->output_buffer; + BufferDesc& vc = nodes[c.b3]->output_buffer; + uint64_t* km = reinterpret_cast(kc.get_data()); + uint64_t* vm = reinterpret_cast(vc.get_data()); + if (!km || !vm) return false; + size_t clen = km[0], mx = km[1], sink = km[4]; + uint32_t hd = c.u1 & 0xFFFFu, nqh = c.u0; + if (km[2] != 1 || vm[3] != km[3] || hd == 0 || hd > 512u) return false; + bool has_new = (c.u1 & 0x10000u) != 0; + size_t ng = (hd + 31) / 32; + size_t win = anchor.params.window_size; + bool sliding = win > 0; + if (has_new && !sliding && clen >= mx) { + size_t ceiling = nodes[c.b2]->params.max_cache_seq_len; + if (!cactus_kv_cache_grow(kc, clen + 1, ceiling) + || !cactus_kv_cache_grow(vc, clen + 1, ceiling)) return false; + km = reinterpret_cast(kc.get_data()); + vm = reinterpret_cast(vc.get_data()); + if (!km || !vm) return false; + mx = km[1]; + } + uint32_t Wn = sliding ? (uint32_t)(mx - sink - 1) : 0u; + uint32_t Sn = sliding ? (uint32_t)sink : 0u; + uint32_t Rn = (Wn > Sn) ? (Wn - Sn) : 1u; + char* kb = (char*)kc.get_data(); + char* vb = (char*)vc.get_data(); + size_t slot = 0, kv_end; + if (has_new) { + bool wrap = sliding && clen >= (size_t)Wn; + slot = wrap ? (size_t)(Sn + ((clen - Sn) % Rn)) : clen; + kv_end = wrap ? (size_t)Wn : clen + 1; + } else { + kv_end = (sliding && clen > (size_t)Wn) ? (size_t)Wn : clen; + } + bool ok = cactus_metal_encode_attention_fused_i8( + anchor.output_buffer.get_data(), data(c.a0), + has_new ? data(c.a1) : nullptr, has_new ? data(c.a2) : nullptr, + kb + 64, vb + 64, kb + 64 + mx * hd, vb + 64 + mx * hd, + data(c.a3), has_new ? data(c.a4) : nullptr, has_new ? data(c.a5) : nullptr, + data(c.b0), data(c.b1), + nqh, hd, hd, + 0u, (uint32_t)kv_end, (uint32_t)slot, has_new ? 1u : 0u, + c.f0, c.f1, + mx * hd, mx * hd, mx * ng * sizeof(float), mx * ng * sizeof(float)); + if (!ok) return false; + if (has_new) { km[0] = clen + 1; vm[0] = clen + 1; } + return true; + } + case 2: { + size_t mm[3] = { c.b0, c.b1, c.b2 }; + CactusQuantMatrix W[3]; + void* outs[3]; + const CactusQuantMatrix* Wp[3]; + for (uint32_t bi = 0; bi < c.u0; ++bi) { + GraphNode& mn = *nodes[mm[bi]]; + auto it = map.find(mn.input_ids[1]); + if (it == map.end()) return false; + W[bi] = nodes[it->second]->output_buffer.to_cq_matrix(); + Wp[bi] = &W[bi]; + outs[bi] = mn.output_buffer.get_data(); + if (!outs[bi] || !c.sc[bi]) return false; + } + const void* x = nodes[c.a0]->output_buffer.get_data(); + if (!x) return false; + void* codes[3] = { c.sc[0], c.sc[1], c.sc[2] }; + if (cactus_metal_encode_transform_batch(x, Wp, (int)c.u0, codes)) { + const void* ccodes[3] = { c.sc[0], c.sc[1], c.sc[2] }; + if (!cactus_metal_encode_gemv_cat(outs, ccodes, Wp, (int)c.u0)) { + for (uint32_t bi = 0; bi < c.u0; ++bi) + if (!cactus_metal_encode_gemv_precoded(outs[bi], c.sc[bi], Wp[bi])) return false; + } + return true; + } + for (uint32_t bi = 0; bi < c.u0; ++bi) + if (!cactus_metal_encode_quant_matmul(outs[bi], x, Wp[bi])) return false; + return true; + } + case 7: { + GraphNode& anchor = *nodes[c.b4]; + GraphNode& down = *nodes[c.b0]; + auto itd = map.find(down.input_ids[1]); + auto itu = map.find(anchor.input_ids[1]); + if (itd == map.end() || itu == map.end() || !c.s0) return false; + CactusQuantMatrix Wd = nodes[itd->second]->output_buffer.to_cq_matrix(); + CactusQuantMatrix Wu = nodes[itu->second]->output_buffer.to_cq_matrix(); + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* row = nodes[c.a1]->output_buffer.get_data(); + if (row && c.u0 == 1u) row = (const char*)row + c.u1; + void* out = anchor.output_buffer.get_data(); + if (!x || !row || !out) return false; + if (!cactus_metal_encode_transform_gemv(c.s0, x, &Wd, row)) return false; + if (!cactus_metal_encode_transform_gemv(out, c.s0, &Wu, nullptr) + && !cactus_metal_encode_quant_matmul(out, c.s0, &Wu)) return false; + return true; + } + case 4: { + GraphNode& anchor = *nodes[c.b4]; + auto it = map.find(anchor.input_ids[1]); + if (it == map.end() || !c.s0) return false; + CactusQuantMatrix W = nodes[it->second]->output_buffer.to_cq_matrix(); + void* out = anchor.output_buffer.get_data(); + const void* gate = nodes[c.a0]->output_buffer.get_data(); + const void* up = nodes[c.a1]->output_buffer.get_data(); + if (!out || !gate || !up) return false; + if (cactus_metal_encode_swiglu_transform(c.s0, gate, up, &W, c.f0) + && cactus_metal_encode_gemv_precoded(out, c.s0, &W)) return true; + size_t M = W.K; + if (!cactus_metal_encode_swiglu(c.s0, gate, up, M, c.f0)) return false; + return cactus_metal_encode_quant_matmul(out, c.s0, &W); + } + case 3: { + GraphNode& anchor = *nodes[c.b4]; + const BufferDesc& src = nodes[c.a0]->output_buffer; + size_t dim = anchor.output_buffer.shape.empty() ? 0 : anchor.output_buffer.shape.back(); + if (dim == 0) return false; + size_t rows = src.total_size / dim; + float out_scale = 1.0f; + if (c.u1 == 1u) { + const void* sp = nodes[c.b1]->output_buffer.get_data(); + if (!sp) return false; + out_scale = (float)*(const __fp16*)sp; + } + if (c.u0 == 1u && rows == 1) { + GraphNode& nn = *nodes[c.a5]; + if (!nn.output_buffer.get_data()) { + if (!c.s1) return false; + nn.output_buffer.set_external(c.s1); + } + if (cactus_metal_encode_rms_norm_add_rms( + anchor.output_buffer.get_data(), nn.output_buffer.get_data(), + src.get_data(), data(c.a1), data(c.a2), data(c.a4), + rows, dim, c.f0, out_scale)) return true; + } + if (c.u1 == 1u && c.u0 != 1u && rows == 1) { + if (cactus_metal_encode_rms_norm_add_scale( + anchor.output_buffer.get_data(), src.get_data(), data(c.a1), data(c.a2), + rows, dim, c.f0, out_scale)) return true; + return false; + } + if (c.u1 == 1u) return false; + if (c.u0 == 1u) { + GraphNode& nn = *nodes[c.a5]; + if (!nn.output_buffer.get_data()) { + if (!c.s1) return false; + nn.output_buffer.set_external(c.s1); + } + if (!cactus_metal_encode_rms_norm_add( + anchor.output_buffer.get_data(), src.get_data(), data(c.a1), data(c.a2), + rows, dim, c.f0)) return false; + return cactus_metal_encode_rms_norm( + nn.output_buffer.get_data(), anchor.output_buffer.get_data(), data(c.a4), + rows, dim, c.f0); + } + return cactus_metal_encode_rms_norm_add( + anchor.output_buffer.get_data(), src.get_data(), data(c.a1), data(c.a2), + rows, dim, c.f0); + } + case 5: { + GraphNode& anchor = *nodes[c.b4]; + const BufferDesc& wb = nodes[c.a1]->output_buffer; + CactusQuantMatrix W = wb.to_cq_matrix(); + void* lg = anchor.output_buffer.get_data(); + if (!c.s0 || !cactus_metal_encode_quant_matmul_ortho(lg, data(c.a0), c.s0, &W)) return false; + size_t V = anchor.output_buffer.total_size; + if (!cactus_metal_encode_softcap(lg, lg, V, c.f0)) { + cactus_metal_encode_scalar(3, lg, lg, V, c.f0); + cactus_metal_encode_unary(1, lg, lg, V); + cactus_metal_encode_scalar(2, lg, lg, V, c.f0); + } + return cactus_graph_metal_tail(lg, V); + } + case 8: { + GraphNode& anchor = *nodes[c.b4]; + const char* pa = (const char*)nodes[c.a0]->output_buffer.get_data(); + const char* pb = (const char*)nodes[c.a1]->output_buffer.get_data(); + void* out = anchor.output_buffer.get_data(); + if (!pa || !pb || !out) return false; + return cactus_metal_encode_gemm_batch(out, pa + c.a2 * (c.b3 ? 4 : 2), pb + c.a3 * 2, + c.u1, (uint32_t)c.b0, (uint32_t)c.b1, c.u0, (int)c.b2, (int)c.b3); + } + case 9: { + GraphNode& anchor = *nodes[c.b4]; + const char* px = (const char*)nodes[c.a0]->output_buffer.get_data(); + const char* pw = (const char*)nodes[c.a1]->output_buffer.get_data(); + void* out = anchor.output_buffer.get_data(); + if (!px || !pw || !out) return false; + return cactus_metal_encode_conv1d_dw(out, px + c.a2 * 2, pw + c.a3 * 2, + c.u0, c.u1, (uint32_t)c.b0, (uint32_t)c.b1, (uint32_t)c.b2); + } + case 10: { + GraphNode& anchor = *nodes[c.b4]; + GraphNode& tn = *nodes[c.a1]; + const char* pp = (const char*)nodes[c.a0]->output_buffer.get_data(); + void* out = anchor.output_buffer.get_data(); + if (!pp || !out) return false; + long ii = -1; + { + auto it = map.find(tn.input_ids[0]); + if (it == map.end()) return false; + ii = (long)it->second; + } + const auto& ish = nodes[(size_t)ii]->output_buffer.shape; + const auto& osh = tn.output_buffer.shape; + const auto& perm = tn.params.permutation; + uint32_t ndim = (uint32_t)perm.size(); + uint32_t ax = c.u1; + if (ndim == 0 || ndim + 1 > 8 || ish.size() != ndim || osh.size() != ndim || ax > ndim) return false; + size_t istr[8]; + istr[ndim - 1] = 1; + for (size_t d = ndim - 1; d > 0; --d) istr[d - 1] = istr[d] * ish[d]; + uint32_t oshape[8], sstride[8]; + for (uint32_t d = 0; d < ax; ++d) { + oshape[d] = (uint32_t)osh[d]; + sstride[d] = (uint32_t)istr[perm[d]]; + } + oshape[ax] = c.u0; sstride[ax] = (uint32_t)c.b0; + for (uint32_t d = ax; d < ndim; ++d) { + oshape[d + 1] = (uint32_t)osh[d]; + sstride[d + 1] = (uint32_t)istr[perm[d]]; + } + size_t total = (size_t)c.u0 * tn.output_buffer.total_size; + return cactus_metal_encode_strided_copy(out, pp + c.a2 * 2, oshape, sstride, ndim + 1, + (uint32_t)total, 0, nodes[c.a0]->output_buffer.byte_size - c.a2 * 2, anchor.output_buffer.byte_size); + } + case 11: { + GraphNode& anchor = *nodes[c.b4]; + const void* in = nodes[c.a0]->output_buffer.get_data(); + void* out = anchor.output_buffer.get_data(); + if (!in || !out) return false; + const void* sides[3] = { nullptr, nullptr, nullptr }; + size_t side_elems[3] = { 0, 0, 0 }; + size_t side_idx[3] = { c.a1, c.a2, c.a3 }; + for (uint32_t si = 0; si < c.u1; ++si) { + sides[si] = nodes[side_idx[si]]->output_buffer.get_data(); + side_elems[si] = nodes[side_idx[si]]->output_buffer.total_size; + if (!sides[si]) return false; + } + const auto& blob = p->blobs[c.b0]; + size_t inner = anchor.output_buffer.shape.empty() ? 1 : anchor.output_buffer.shape.back(); + return cactus_metal_encode_elemwise_chain(out, in, blob.data(), c.u0, + sides[0], sides[1], sides[2], side_elems, anchor.output_buffer.total_size, + (uint32_t)c.b1, (uint32_t)inner); + } + case 12: { + GraphNode& anchor = *nodes[c.b4]; + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* res = nodes[c.a1]->output_buffer.get_data(); + void* ysum = nodes[c.a2]->output_buffer.get_data(); + const void* w = nodes[c.a3]->output_buffer.get_data(); + void* ynorm = anchor.output_buffer.get_data(); + if (!x || !res || !ysum || !w || !ynorm) return false; + return cactus_metal_encode_rms_norm_add_rows(ysum, ynorm, x, res, w, + c.u0, c.u1, c.f0, (int)c.b0); + } + case 13: { + GraphNode& anchor = *nodes[c.b4]; + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* w = nodes[c.a1]->output_buffer.get_data(); + const void* b = nodes[c.a2]->output_buffer.get_data(); + void* y = anchor.output_buffer.get_data(); + if (!x || !w || !b || !y) return false; + return cactus_metal_encode_gemv_bias(y, x, w, b, c.u0, c.u1, (int)c.b0); + } + case 14: { + GraphNode& anchor = *nodes[c.b4]; + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* cs = nodes[c.a1]->output_buffer.get_data(); + const void* sn = nodes[c.a2]->output_buffer.get_data(); + void* y = anchor.output_buffer.get_data(); + if (!x || !cs || !sn || !y) return false; + return cactus_metal_encode_rope_pair(y, x, cs, sn, c.u0, c.u1); + } + case 15: { + GraphNode& anchor = *nodes[c.b4]; + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* w = nodes[c.a1]->output_buffer.get_data(); + void* y = anchor.output_buffer.get_data(); + if (!x || !w || !y) return false; + return cactus_metal_encode_rms_norm_scale(y, x, w, c.u0, c.u1, c.f0, c.f1); + } + case 16: { + GraphNode& anchor = *nodes[c.b4]; + const void* lg = nodes[c.a0]->output_buffer.get_data(); + void* probs = nodes[c.a1]->output_buffer.get_data(); + void* tk = anchor.output_buffer.get_data(); + if (!lg || !probs || !tk) return false; + return cactus_metal_encode_softmax_topk(probs, tk, lg, c.u0, c.u1, c.b0, c.f0); + } + case 18: { + GraphNode& anchor = *nodes[c.b4]; + const void* a = nodes[c.a0]->output_buffer.get_data(); + const void* wa = nodes[c.a1]->output_buffer.get_data(); + const void* b = nodes[c.a2]->output_buffer.get_data(); + const void* wb = nodes[c.a3]->output_buffer.get_data(); + void* y = anchor.output_buffer.get_data(); + if (!a || !wa || !b || !wb || !y) return false; + return cactus_metal_encode_rms2_add_clip(y, a, wa, b, wb, c.u1, c.f0, c.f1); + } + case 17: { + GraphNode& anchor = *nodes[c.b4]; + const void* x = nodes[c.a0]->output_buffer.get_data(); + const void* w = nodes[c.a3]->output_buffer.get_data(); + const void* cs = nodes[c.a1]->output_buffer.get_data(); + const void* sn = nodes[c.a2]->output_buffer.get_data(); + void* y = anchor.output_buffer.get_data(); + if (!x || !w || !cs || !sn || !y) return false; + return cactus_metal_encode_rope_pair_rms(y, x, w, cs, sn, c.u0, c.u1, c.f0); + } + default: return false; + } +} diff --git a/cactus-graph/src/metal_runtime.cpp b/cactus-graph/src/metal_runtime.cpp new file mode 100644 index 000000000..311dec03a --- /dev/null +++ b/cactus-graph/src/metal_runtime.cpp @@ -0,0 +1,196 @@ +#include "../cactus_graph.h" +#include "cactus_kernels.h" +#include "metal_backend.h" + +#include +#include +#include + +namespace { + +static bool g_metal_argmax_valid = false; +static const float* g_metal_argmax_buf = nullptr; + +} + +static bool g_last_step_adjusted = false; +static bool g_last_amax_biased = false; +static bool g_prefill_consistent = false; + +void cactus_graph_mark_unadjusted() { + g_last_step_adjusted = false; + g_last_amax_biased = false; + g_metal_argmax_valid = false; +} + +void cactus_graph_set_prefill_consistent(bool on) { g_prefill_consistent = on; } + +bool cactus_graph_prefill_consistent() { return g_prefill_consistent; } + +struct GSampling { + bool active = false; + float rep_penalty = 1.0f; + std::vector recent; + const float* bias_dense = nullptr; + size_t bias_len = 0; + long long suppressed = -1; +}; +static GSampling g_samp; +static float* g_bias_dense = nullptr; +static size_t g_bias_dense_n = 0; + +void cactus_graph_set_sampling(const uint32_t* recent, int n_recent, float rep_penalty, + const float* bias_dense, size_t bias_len, + long long suppressed) { + g_samp.active = true; + g_samp.rep_penalty = rep_penalty; + g_samp.recent.assign(recent, recent + (n_recent > 0 ? n_recent : 0)); + g_samp.bias_dense = bias_dense; + g_samp.bias_len = bias_dense ? bias_len : 0; + g_samp.suppressed = suppressed; +} +void cactus_graph_clear_sampling() { g_samp = GSampling(); } +bool cactus_graph_metal_adjusted() { return g_last_step_adjusted; } +bool cactus_graph_metal_argmax_biased() { return g_last_amax_biased; } + +static void* g_plan_amax = nullptr; +static bool g_plan_tail_pending = false; +static bool g_plan_tail_adjusted = false; +static bool g_plan_tail_biased = false; + + +static const float* update_bias_dense(size_t vocab) { + if (!g_samp.bias_dense || g_samp.bias_len == 0) return nullptr; + if (!g_bias_dense || g_bias_dense_n < vocab) { + if (g_bias_dense) cactus_metal_free_shared(g_bias_dense); + g_bias_dense = (float*)cactus_metal_alloc_shared(vocab * sizeof(float)); + if (!g_bias_dense) { g_bias_dense_n = 0; return nullptr; } + g_bias_dense_n = vocab; + } + size_t n = g_samp.bias_len < vocab ? g_samp.bias_len : vocab; + std::memcpy(g_bias_dense, g_samp.bias_dense, n * sizeof(float)); + if (n < vocab) std::memset(g_bias_dense + n, 0, (vocab - n) * sizeof(float)); + return g_bias_dense; +} + +bool cactus_graph_metal_tail(void* logits, size_t vocab) { + g_plan_tail_pending = false; + bool adjusted = false; + if (g_samp.active && + ((g_samp.rep_penalty != 1.0f && !g_samp.recent.empty()) || g_samp.suppressed >= 0)) { + adjusted = cactus_metal_encode_adjust_logits(logits, vocab, + g_samp.recent.data(), (uint32_t)g_samp.recent.size(), + g_samp.suppressed, g_samp.rep_penalty); + if (!adjusted) return false; + } + if (!g_plan_amax) { + g_plan_amax = cactus_metal_alloc_shared(3 * sizeof(float)); + if (!g_plan_amax) return false; + } + const float* bias = g_samp.active ? update_bias_dense(vocab) : nullptr; + if (!cactus_metal_encode_argmax(logits, (uint32_t)vocab, g_plan_amax, bias)) return false; + g_plan_tail_pending = true; + g_plan_tail_adjusted = adjusted; + g_plan_tail_biased = (bias != nullptr); + return true; +} + +void cactus_graph_metal_tail_commit() { + if (!g_plan_tail_pending) return; + g_plan_tail_pending = false; + g_metal_argmax_buf = (const float*)g_plan_amax; + g_metal_argmax_valid = true; + g_last_step_adjusted = g_plan_tail_adjusted; + g_last_amax_biased = g_plan_tail_biased; +} + + +bool cactus_graph_metal_argmax(uint32_t* idx, float* best, float* second) { + if (!g_metal_argmax_valid || !g_metal_argmax_buf) return false; + g_metal_argmax_valid = false; + *best = g_metal_argmax_buf[0]; *second = g_metal_argmax_buf[1]; + *idx = (uint32_t)g_metal_argmax_buf[2]; + return true; +} + +static FusedEmbedCtx g_fe; +void cactus_graph_set_fused_embed(const FusedEmbedCtx* ctx) { + if (ctx && ctx->ok) g_fe = *ctx; else g_fe.ok = false; +} +const FusedEmbedCtx* cactus_graph_fused_embed() { return g_fe.ok ? &g_fe : nullptr; } + +bool cactus_graph_metal_fold_prologue(void* h_buf, void* ple_buf, void* pos_buf, + const CactusQuantMatrix* lm_head, size_t nl, size_t ple_dim) { + if (!g_fe.ok || !lm_head) return false; + static void* pe = nullptr; + static void* pa = nullptr; + static void* pj = nullptr; + static void* pjs = nullptr; + static size_t cap = 0; + static size_t cap_pe = 0; + const size_t PK = g_fe.proj.N; + const size_t EK = g_fe.ple.K; + if (cap_pe < EK) { + if (pe) cactus_metal_free_shared(pe); + pe = cactus_metal_alloc_shared(EK * 2); + cap_pe = pe ? EK : 0; + } + if (cap < PK) { + if (pa) cactus_metal_free_shared(pa); + if (pj) cactus_metal_free_shared(pj); + if (pjs) cactus_metal_free_shared(pjs); + pa = cactus_metal_alloc_shared(PK * 2); + pj = cactus_metal_alloc_shared(PK * 2); + pjs = cactus_metal_alloc_shared(PK * 2); + cap = (pa && pj && pjs) ? PK : 0; + } + if (!pe || !pa || !pj || !pjs) return false; + uint32_t tok = (uint32_t)g_fe.token_id; + if (!cactus_metal_encode_embedding_ortho(h_buf, tok, lm_head, g_fe.emb_scale)) return false; + if (!cactus_metal_encode_embedding_hadamard(pe, tok, &g_fe.ple)) return false; + cactus_metal_encode_scalar(2, pa, pe, PK, g_fe.ple_scale); + if (!cactus_metal_encode_transform_gemv(pj, h_buf, &g_fe.proj, nullptr) + && !cactus_metal_encode_quant_matmul(pj, h_buf, &g_fe.proj)) return false; + cactus_metal_encode_scalar(2, pjs, pj, PK, g_fe.proj_scale); + if (!cactus_metal_encode_rms_norm_add_scale(ple_buf, pjs, g_fe.rms_weight, pa, nl, ple_dim, g_fe.rms_eps, g_fe.final_scale)) { + if (!cactus_metal_encode_rms_norm_add(pa, pjs, g_fe.rms_weight, pa, nl, ple_dim, g_fe.rms_eps)) return false; + cactus_metal_encode_scalar(2, ple_buf, pa, PK, g_fe.final_scale); + } + if (pos_buf) *(float*)pos_buf = (float)g_fe.position; + return true; +} + +bool CactusGraph::extract_ple_pathway(FusedEmbedCtx& ctx) const { + + if (nodes_.size() != 23) return false; + auto op = [&](size_t i){ return nodes_[i]->op_type; }; + if (op(3) != OpType::INPUT || op(4) != OpType::INPUT || op(5) != OpType::INPUT) return false; + if (op(8) != OpType::EMBEDDING || op(12) != OpType::MATMUL || op(19) != OpType::RMS_NORM) return false; + if (op(7) != OpType::SCALAR_MULTIPLY || op(9) != OpType::SCALAR_MULTIPLY || + op(14) != OpType::SCALAR_MULTIPLY || op(22) != OpType::SCALAR_MULTIPLY) return false; + const BufferDesc& pleW = nodes_[3]->output_buffer; + const BufferDesc& projW = nodes_[5]->output_buffer; + if (!pleW.is_cq() || (pleW.cq_flags & CACTUS_QUANT_FLAG_ORTHOGONAL) || !projW.is_cq()) return false; + ctx.ple = pleW.to_cq_matrix(); + ctx.proj = projW.to_cq_matrix(); + ctx.rms_weight = nodes_[4]->output_buffer.get_data(); + ctx.emb_scale = nodes_[7]->params.scalar; + ctx.ple_scale = nodes_[9]->params.scalar; + ctx.proj_scale = nodes_[14]->params.scalar; + ctx.final_scale = nodes_[22]->params.scalar; + ctx.rms_eps = nodes_[19]->params.epsilon; + if (!ctx.rms_weight || ctx.emb_scale == 0.0f || ctx.proj.N == 0) return false; + ctx.ok = true; + return true; +} + +void cactus_graph_on_destroy(const void* graph) { + (void)graph; + g_samp = GSampling(); + cactus_metal_invalidate_host_wraps(); +} + +CactusGraph::~CactusGraph() { + cactus_graph_on_destroy(this); + invalidate_metal_state(); +} diff --git a/cactus-graph/src/ops_cache.cpp b/cactus-graph/src/ops_cache.cpp index fc2da274f..f0ecd9d78 100644 --- a/cactus-graph/src/ops_cache.cpp +++ b/cactus-graph/src/ops_cache.cpp @@ -1,5 +1,6 @@ #include "../cactus_graph.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include #include #include @@ -101,6 +102,10 @@ inline bool use_fp16_kv_cache() { constexpr size_t kInitialCacheEntries = 256; +inline bool kv_cache_resident() { + return cactus_backend_metal(); +} + inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { auto* meta = get_meta(buf); size_t cur = meta->max_seq_len; @@ -114,7 +119,15 @@ inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { size_t total = fp16_cache ? fp16_cache_elements(new_max, kv_heads, hdim) : cache_buffer_size(new_max, kv_heads, hdim); BufferDesc resized({total}, fp16_cache ? Precision::FP16 : Precision::INT8); - resized.allocate(); + const bool metal = kv_cache_resident(); + void* old_data = buf.get_data(); + if (metal) { + cactus_metal_session_sync(); + void* p = cactus_metal_alloc_shared(resized.byte_size); + if (p) resized.set_external(p); else resized.allocate(); + } else { + resized.allocate(); + } std::memset(resized.get_data(), 0, resized.byte_size); std::memcpy(resized.get_data(), buf.get_data(), sizeof(CacheMetadata)); @@ -130,6 +143,7 @@ inline bool resize_cache_buffer(BufferDesc& buf, size_t new_max) { } get_meta(resized)->max_seq_len = new_max; buf = std::move(resized); + cactus_metal_free_shared(old_data); return true; } @@ -145,6 +159,10 @@ inline bool grow_cache_buffer(BufferDesc& buf, size_t needed, size_t ceiling) { } // namespace +bool cactus_kv_cache_grow(BufferDesc& buf, size_t needed, size_t ceiling) { + return grow_cache_buffer(buf, needed, ceiling); +} + void compute_kv_cache_state_node( GraphNode& node, const nodes_vector&, @@ -158,7 +176,7 @@ void compute_kv_cache_state_node( bool sliding = window > 0 && window < ceiling; size_t max_seq; if (sliding) max_seq = std::min(ceiling, window + node.params.cache_sink_size + 1); - else if (num_slots > 1) max_seq = ceiling; + else if (num_slots > 1 || window > 0) max_seq = ceiling; else max_seq = std::min(ceiling, kInitialCacheEntries); size_t kv_heads = node.params.num_kv_heads; size_t hdim = node.params.head_dim; @@ -168,7 +186,12 @@ void compute_kv_cache_state_node( : cache_buffer_size(max_seq, kv_heads, hdim); node.output_buffer = BufferDesc({num_slots * per_slot}, fp16_cache ? Precision::FP16 : Precision::INT8); - node.output_buffer.allocate(); + if (kv_cache_resident()) { + void* p = cactus_metal_alloc_shared(node.output_buffer.byte_size); + if (p) node.output_buffer.set_external(p); else node.output_buffer.allocate(); + } else { + node.output_buffer.allocate(); + } std::memset(node.output_buffer.get_data(), 0, node.output_buffer.byte_size); auto* meta0 = get_meta(node.output_buffer, 0); @@ -592,8 +615,15 @@ struct PaddedAppendBackup { uint64_t overshoot; uint64_t keep_sink; uint64_t kept_padded; + uint64_t ring_rows; }; +size_t ring_row_for_pos(size_t pos, size_t sink, size_t ring_capacity) { + if (pos < ring_capacity) return pos; + size_t recent = ring_capacity > sink ? ring_capacity - sink : 1; + return sink + (pos - sink) % recent; +} + struct CacheRowRegion { size_t offset; size_t row_bytes; @@ -625,6 +655,27 @@ std::vector CactusGraph::snapshot_cache_padded_append(size_t node_id, s const auto* meta = get_meta(buf); const size_t len0 = meta->current_seq_len; const size_t appended = real_tokens + pad_tokens; + if (kv_cache_resident()) { + const size_t ring_window = meta->max_seq_len - meta->sink_size - 1; + if (ring_window <= meta->sink_size || appended >= ring_window - meta->sink_size) { + throw std::runtime_error("padded cache append larger than the attention window is not supported"); + } + std::vector backup(sizeof(PaddedAppendBackup)); + auto* header = reinterpret_cast(backup.data()); + header->overshoot = 0; + header->keep_sink = meta->sink_size; + header->kept_padded = ring_window; + header->ring_rows = pad_tokens; + const auto* base = static_cast(buf.get_data()); + for (const auto& region : cache_row_regions(buf, meta)) { + for (size_t t = 0; t < pad_tokens; ++t) { + size_t row = ring_row_for_pos(len0 + real_tokens + t, meta->sink_size, ring_window); + const uint8_t* src = base + region.offset + row * region.row_bytes; + backup.insert(backup.end(), src, src + region.row_bytes); + } + } + return backup; + } if (len0 == 0 || len0 + appended <= window) return {}; const size_t keep_sink = std::min({static_cast(meta->sink_size), len0, window}); const size_t tail_capacity = window - keep_sink; @@ -662,6 +713,23 @@ void CactusGraph::rollback_cache_padded_append(size_t node_id, size_t real_token return; } const auto* header = reinterpret_cast(backup.data()); + if (header->ring_rows > 0) { + const size_t ring_window = header->kept_padded; + const size_t sink = header->keep_sink; + const size_t len_after = meta->current_seq_len; + const size_t len0 = len_after >= real_tokens + pad_tokens ? len_after - real_tokens - pad_tokens : 0; + auto* base = static_cast(buf.get_data()); + const uint8_t* saved = backup.data() + sizeof(PaddedAppendBackup); + for (const auto& region : cache_row_regions(buf, meta)) { + for (size_t t = 0; t < header->ring_rows; ++t) { + size_t row = ring_row_for_pos(len0 + real_tokens + t, sink, ring_window); + std::memcpy(base + region.offset + row * region.row_bytes, saved, region.row_bytes); + saved += region.row_bytes; + } + } + meta->current_seq_len = len0 + real_tokens; + return; + } const size_t overshoot = header->overshoot; const size_t keep_sink = header->keep_sink; const size_t kept_padded = header->kept_padded; diff --git a/cactus-graph/src/ops_conv.cpp b/cactus-graph/src/ops_conv.cpp index 1033b01a0..d17f1340d 100644 --- a/cactus-graph/src/ops_conv.cpp +++ b/cactus-graph/src/ops_conv.cpp @@ -7,7 +7,6 @@ #include #include -namespace { std::vector<__fp16> dequantize_int8_weights_to_fp16( const BufferDesc& W, size_t rows, @@ -40,7 +39,6 @@ std::vector<__fp16> dequantize_int8_weights_to_fp16( } return W_fp16; } -} void compute_conv1d_causal_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { if (node.params.backend == ComputeBackend::NPU) { diff --git a/cactus-graph/src/ops_math.cpp b/cactus-graph/src/ops_math.cpp index e87fc110b..e12661268 100644 --- a/cactus-graph/src/ops_math.cpp +++ b/cactus-graph/src/ops_math.cpp @@ -1,5 +1,6 @@ #include "../cactus_graph.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include #include #include @@ -594,14 +595,22 @@ void compute_reshape_node(GraphNode& node, const std::vector(input_buffer.get_data())); + } else { + std::memcpy(node.output_buffer.get_data(), input_buffer.get_data(), input_buffer.byte_size); + } } void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { const auto& input_buf = get_input(node, 0, nodes, node_index_map); if (input_buf.precision == node.output_buffer.precision) { - std::memcpy(node.output_buffer.get_data(), input_buf.get_data(), input_buf.byte_size); + if (cactus_metal_active_mode()) { + node.output_buffer.set_external(const_cast(input_buf.get_data())); + } else { + std::memcpy(node.output_buffer.get_data(), input_buf.get_data(), input_buf.byte_size); + } return; } diff --git a/cactus-graph/src/ops_nn.cpp b/cactus-graph/src/ops_nn.cpp index bee9810ba..e0ad4de91 100644 --- a/cactus-graph/src/ops_nn.cpp +++ b/cactus-graph/src/ops_nn.cpp @@ -1,5 +1,6 @@ #include "../cactus_graph.h" #include "cactus_kernels.h" +#include "metal_backend.h" #include #include #include @@ -56,6 +57,8 @@ void compute_matmul_node(GraphNode& node, const std::vector(M), output); + else if (cactus_backend_metal()) + cactus_metal_quant_matmul(&mat, lhs, static_cast(M), output); else cactus_quant_matmul(&mat, lhs, static_cast(M), output); } else { diff --git a/cactus-graph/src/ops_tensor.cpp b/cactus-graph/src/ops_tensor.cpp index 857c677b9..c33a59996 100644 --- a/cactus-graph/src/ops_tensor.cpp +++ b/cactus-graph/src/ops_tensor.cpp @@ -76,15 +76,15 @@ void compute_slice_node(GraphNode& node, const std::vector(input_buffer.get_data()); if (!base_ptr) { throw std::runtime_error("Slice input buffer is not available"); } - const size_t byte_offset = PrecisionTraits::byte_offset_of(input_buffer.precision, slice_start * dims0.inner); + const size_t byte_offset = PrecisionTraits::byte_offset_of(input_buffer.precision, slice_start * dims.inner); node.output_buffer.set_external(base_ptr + byte_offset); node.output_buffer.precision = input_buffer.precision; @@ -101,8 +101,6 @@ void compute_slice_node(GraphNode& node, const std::vector +#include +#include +#include +#include +#include +#include + +using namespace TestUtils; + +namespace { + +std::vector<__fp16> random_fp16(size_t n, float lo = -1.0f, float hi = 1.0f, uint32_t seed = 42) { + std::mt19937 gen(seed); + std::uniform_real_distribution dis(lo, hi); + std::vector<__fp16> out(n); + for (auto& v : out) v = static_cast<__fp16>(dis(gen)); + return out; +} + +struct ParityCase { + std::vector> input_shapes; + std::vector> input_data; + std::vector> input_data_f32; + std::function&)> build; + float tolerance = 5e-2f; + + size_t add_input(const std::vector& shape, float lo = -1.0f, float hi = 1.0f) { + size_t n = 1; + for (size_t d : shape) n *= d; + input_shapes.push_back(shape); + input_data.push_back(random_fp16(n, lo, hi, 42 + (uint32_t)input_shapes.size())); + input_data_f32.push_back({}); + return input_shapes.size() - 1; + } + + size_t add_input_data(const std::vector& shape, std::vector<__fp16> data) { + input_shapes.push_back(shape); + input_data.push_back(std::move(data)); + input_data_f32.push_back({}); + return input_shapes.size() - 1; + } + + size_t add_input_f32(const std::vector& shape, const std::vector& data) { + input_shapes.push_back(shape); + input_data.push_back({}); + input_data_f32.push_back(data); + return input_shapes.size() - 1; + } + + std::vector run(const char* backend) { + if (cactus_backend_select(backend) != 0) return {}; + CactusGraph graph; + std::vector ids; + for (size_t i = 0; i < input_shapes.size(); ++i) { + bool f32 = !input_data_f32[i].empty(); + ids.push_back(graph.input(input_shapes[i], f32 ? Precision::FP32 : Precision::FP16)); + } + size_t out_id = build(graph, ids); + for (size_t i = 0; i < ids.size(); ++i) { + bool f32 = !input_data_f32[i].empty(); + graph.set_input(ids[i], + f32 ? static_cast(input_data_f32[i].data()) + : static_cast(input_data[i].data()), + f32 ? Precision::FP32 : Precision::FP16); + } + graph.execute(); + const auto& desc = graph.get_output_buffer(out_id); + std::vector result(desc.total_size); + if (desc.precision == Precision::FP16) { + const __fp16* p = static_cast(graph.get_output(out_id)); + for (size_t i = 0; i < result.size(); ++i) result[i] = static_cast(p[i]); + } else { + const float* p = static_cast(graph.get_output(out_id)); + for (size_t i = 0; i < result.size(); ++i) result[i] = p[i]; + } + graph.hard_reset(); + cactus_backend_select("auto"); + return result; + } + + bool check() { + std::vector cpu = run("cpu"); + std::vector metal = run("metal"); + if (cpu.size() != metal.size() || cpu.empty()) return false; + for (size_t i = 0; i < cpu.size(); ++i) { + float scale = std::max(1.0f, std::fabs(cpu[i])); + if (std::fabs(cpu[i] - metal[i]) > tolerance * scale) { + std::cout << " mismatch at " << i << ": cpu=" << cpu[i] + << " metal=" << metal[i] << "\n"; + return false; + } + } + return true; + } +}; + +bool metal_present() { + bool ok = cactus_backend_select("metal") == 0; + cactus_backend_select("auto"); + return ok; +} + +bool parity_unary(size_t (CactusGraph::*fn)(size_t), float lo = -1.0f, float hi = 1.0f) { + ParityCase c; + c.add_input({4, 33}, lo, hi); + c.build = [fn](CactusGraph& g, const std::vector& in) { return (g.*fn)(in[0]); }; + return c.check(); +} + +bool parity_scalar(size_t (CactusGraph::*fn)(size_t, float), float p, float lo = -1.0f, float hi = 1.0f) { + ParityCase c; + c.add_input({4, 33}, lo, hi); + c.build = [fn, p](CactusGraph& g, const std::vector& in) { return (g.*fn)(in[0], p); }; + return c.check(); +} + +bool parity_reduce(size_t (CactusGraph::*fn)(size_t, int)) { + for (int axis = 0; axis < 3; ++axis) { + ParityCase c; + c.add_input({3, 5, 17}); + c.build = [fn, axis](CactusGraph& g, const std::vector& in) { return (g.*fn)(in[0], axis); }; + if (!c.check()) return false; + } + return true; +} + +bool parity_not_equal() { + ParityCase c; + c.add_input({2, 31}); + c.add_input({2, 31}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.not_equal(in[0], in[1]); }; + return c.check(); +} + +bool parity_broadcast_binary() { + { + ParityCase c; + c.add_input({4, 33}); + c.add_input({1, 33}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.multiply(in[0], in[1]); }; + if (!c.check()) return false; + } + { + ParityCase c; + c.add_input({4, 33}); + c.add_input({1, 33}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.not_equal(in[0], in[1]); }; + if (!c.check()) return false; + } + return true; +} + +bool parity_concat() { + for (int axis = 0; axis < 2; ++axis) { + ParityCase c; + c.add_input({3, 7}); + c.add_input({axis == 0 ? 5u : 3u, axis == 0 ? 7u : 11u}); + c.build = [axis](CactusGraph& g, const std::vector& in) { return g.concat(in[0], in[1], axis); }; + if (!c.check()) return false; + } + return true; +} + +bool parity_gather() { + ParityCase c; + c.add_input({16, 24}); + c.add_input_f32({6}, {3.0f, 0.0f, 15.0f, 7.0f, 7.0f, 2.0f}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.gather(in[0], in[1]); }; + return c.check(); +} + +bool parity_rope(bool gptj) { + ParityCase c; + c.add_input({1, 6, 4, 32}); + c.build = [gptj](CactusGraph& g, const std::vector& in) { + return gptj ? g.rope_gptj(in[0], 10000.0f, 3, 16) : g.rope(in[0], 10000.0f, 3); + }; + return c.check(); +} + +bool parity_maxpool1d() { + ParityCase c; + c.add_input({2, 5, 29}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.maxpool1d(in[0], 3, 2); }; + return c.check(); +} + +bool parity_bilinear() { + ParityCase c; + c.add_input({64, 12}); + c.build = [](CactusGraph& g, const std::vector& in) { + return g.bilinear_interpolation(in[0], 13, 11, false); + }; + return c.check(); +} + +bool parity_conv1d() { + ParityCase c; + c.add_input({2, 6, 31}); + c.add_input({4, 6, 5}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.conv1d(in[0], in[1], 2); }; + return c.check(); +} + +bool parity_conv1d_k7s3() { + ParityCase c; + c.add_input({1, 8, 40}); + c.add_input({8, 7, 16}); + c.add_input({16}); + c.build = [](CactusGraph& g, const std::vector& in) { return g.conv1d_k7s3(in[0], in[1], in[2]); }; + return c.check(); +} + +bool parity_conv1d_causal() { + for (size_t dil : {1u, 2u}) { + ParityCase c; + c.add_input({1, 21, 8}); + c.add_input({8, 1, 4}); + c.build = [dil](CactusGraph& g, const std::vector& in) { + return g.conv1d_causal(in[0], in[1], 4, dil); + }; + if (!c.check()) return false; + } + return true; +} + +bool parity_conv1d_dw_k9() { + ParityCase c; + c.add_input({1, 25, 6}); + c.add_input({6, 1, 9}); + c.add_input({6}); + c.build = [](CactusGraph& g, const std::vector& in) { + return g.conv1d_same_depthwise_k9(in[0], in[1], in[2]); + }; + return c.check(); +} + +bool parity_conv1d_pointwise() { + ParityCase c; + c.add_input({1, 13, 16}); + c.add_input({24, 16}); + c.add_input({24}); + c.build = [](CactusGraph& g, const std::vector& in) { + return g.conv1d_pointwise(in[0], in[1], in[2]); + }; + return c.check(); +} + +bool parity_conv2d(size_t (CactusGraph::*fn)(size_t, size_t, size_t), bool depthwise, bool pointwise) { + ParityCase c; + c.add_input({1, 4, 11, 13}); + if (depthwise) c.add_input({4, 1, 3, 3}); + else if (pointwise) c.add_input({6, 4, 1, 1}); + else c.add_input({6, 4, 3, 3}); + c.add_input({pointwise || depthwise ? (depthwise ? 4u : 6u) : 6u}); + c.build = [fn](CactusGraph& g, const std::vector& in) { return (g.*fn)(in[0], in[1], in[2]); }; + return c.check(); +} + +bool parity_batchnorm() { + ParityCase c; + c.add_input({2, 5, 9}); + c.add_input({5}); + c.add_input({5}); + c.add_input({5}); + c.add_input({5}, 0.5f, 1.5f); + c.build = [](CactusGraph& g, const std::vector& in) { + return g.batchnorm(in[0], in[1], in[2], in[3], in[4], 1); + }; + return c.check(); +} + +bool parity_groupnorm() { + ParityCase c; + c.add_input({2, 8, 6}); + c.add_input({8}); + c.add_input({8}); + c.build = [](CactusGraph& g, const std::vector& in) { + return g.groupnorm(in[0], in[1], in[2], 4); + }; + return c.check(); +} + +bool parity_cumsum() { + for (int axis = 0; axis < 2; ++axis) { + ParityCase c; + c.add_input({4, 19}); + c.build = [axis](CactusGraph& g, const std::vector& in) { return g.cumsum(in[0], axis); }; + if (!c.check()) return false; + } + return true; +} + +bool parity_softmax() { + ParityCase c; + c.add_input({5, 333}, -4.0f, 4.0f); + c.tolerance = 5e-3f; + c.build = [](CactusGraph& g, const std::vector& in) { return g.softmax(in[0], -1); }; + return c.check(); +} + +bool parity_rms_norm() { + for (size_t rows : {1u, 7u}) { + ParityCase c; + c.add_input({rows, 256}); + c.add_input({256}, 0.5f, 1.5f); + c.tolerance = 1e-2f; + c.build = [](CactusGraph& g, const std::vector& in) { + return g.rms_norm(in[0], in[1], 1e-5f); + }; + if (!c.check()) return false; + } + return true; +} + +bool parity_residual_rms_norm() { + for (size_t rows : {1u, 6u}) { + ParityCase c; + c.add_input({rows, 192}); + c.add_input({rows, 192}); + c.add_input({192}, 0.5f, 1.5f); + c.tolerance = 1e-2f; + c.build = [](CactusGraph& g, const std::vector& in) { + return g.rms_norm(g.add(in[0], in[1]), in[2], 1e-5f); + }; + if (!c.check()) return false; + } + return true; +} + +bool parity_elemwise_chain() { + ParityCase c; + c.add_input({4, 96}); + c.add_input({4, 96}); + c.add_input({4, 96}); + c.tolerance = 1e-2f; + c.build = [](CactusGraph& g, const std::vector& in) { + return g.add(g.multiply(g.silu(in[0]), in[1]), g.scalar_multiply(in[2], 0.5f)); + }; + return c.check(); +} + +bool parity_flash_attention() { + for (size_t T : {64u, 89u, 121u}) { + ParityCase c; + const size_t H = 2, D = 64, S = 729; + c.add_input({1, T, H, D}); + c.add_input({1, S, H, D}); + c.add_input({1, S, H, D}); + c.tolerance = 2e-2f; + c.build = [D](CactusGraph& g, const std::vector& in) { + return g.attention(in[0], in[1], in[2], 1.0f / std::sqrt((float)D), false); + }; + if (!c.check()) return false; + } + return true; +} + +bool parity_attention_causal() { + ParityCase c; + const size_t T = 37, H = 4, HKV = 2, D = 32; + c.add_input({1, T, H, D}); + c.add_input({1, T, HKV, D}); + c.add_input({1, T, HKV, D}); + c.tolerance = 2e-2f; + c.build = [D](CactusGraph& g, const std::vector& in) { + return g.attention(in[0], in[1], in[2], 1.0f / std::sqrt((float)D), true); + }; + return c.check(); +} + +bool parity_equality_exact() { + const size_t n = 2 * 31; + std::vector<__fp16> a = random_fp16(n, -1.0f, 1.0f, 7); + std::vector<__fp16> b = random_fp16(n, -1.0f, 1.0f, 8); + for (size_t i = 0; i < n; i += 2) b[i] = a[i]; + { + ParityCase c; + c.add_input_data({2, 31}, a); + c.add_input_data({2, 31}, b); + c.tolerance = 1e-3f; + c.build = [](CactusGraph& g, const std::vector& in) { return g.not_equal(in[0], in[1]); }; + if (!c.check()) return false; + } + { + std::vector<__fp16> z = a; + for (size_t i = 0; i < n; i += 2) z[i] = (__fp16)0.0f; + ParityCase c; + c.add_input_data({2, 31}, z); + c.tolerance = 1e-3f; + c.build = [](CactusGraph& g, const std::vector& in) { return g.scalar_not_equal(in[0], 0.0f); }; + if (!c.check()) return false; + } + return true; +} + +struct CacheStepData { + std::vector<__fp16> q, k, v; + size_t tokens; +}; + +std::vector run_cached_attention(const char* backend, size_t ceiling, size_t window, size_t sink, + size_t h, size_t kv, size_t d, + const std::vector& steps, size_t collect_from) { + if (cactus_backend_select(backend) != 0) return {}; + const size_t sentinel = std::numeric_limits::max(); + const float scale = 1.0f / std::sqrt((float)d); + std::vector collected; + CactusGraph g; + size_t kc = g.kv_cache_state(ceiling, kv, d, window, sink); + size_t vc = g.kv_cache_state(ceiling, kv, d, window, sink); + for (size_t si = 0; si < steps.size(); ++si) { + const auto& data = steps[si]; + size_t s = data.tokens; + size_t iq = g.input({1, s, h, d}, Precision::FP16); + size_t ik = g.input({1, s, kv, d}, Precision::FP16); + size_t iv = g.input({1, s, kv, d}, Precision::FP16); + g.set_input(iq, const_cast<__fp16*>(data.q.data()), Precision::FP16); + g.set_input(ik, const_cast<__fp16*>(data.k.data()), Precision::FP16); + g.set_input(iv, const_cast<__fp16*>(data.v.data()), Precision::FP16); + g.kv_cache_append(ik, kc, window, sink); + g.kv_cache_append(iv, vc, window, sink); + size_t attn = g.attention_cached(iq, ik, iv, kc, vc, scale, sentinel, window); + g.execute(); + if (si >= collect_from) { + const __fp16* p = static_cast(g.get_output(attn)); + for (size_t i = 0; i < s * h * d; ++i) collected.push_back((float)p[i]); + } + g.soft_reset(); + } + g.hard_reset(); + cactus_backend_select("auto"); + return collected; +} + +bool check_cached_attention(size_t ceiling, size_t window, size_t sink, + const std::vector& chunk_sizes, size_t decode_steps, + float tolerance, size_t collect_from = 0) { + const size_t h = 4, kv = 2, d = 32; + std::vector steps; + uint32_t seed = 100; + for (size_t s : chunk_sizes) { + steps.push_back({random_fp16(s * h * d, -1.0f, 1.0f, seed), + random_fp16(s * kv * d, -1.0f, 1.0f, seed + 1), + random_fp16(s * kv * d, -1.0f, 1.0f, seed + 2), s}); + seed += 3; + } + for (size_t i = 0; i < decode_steps; ++i) { + steps.push_back({random_fp16(h * d, -1.0f, 1.0f, seed), + random_fp16(kv * d, -1.0f, 1.0f, seed + 1), + random_fp16(kv * d, -1.0f, 1.0f, seed + 2), 1}); + seed += 3; + } + std::vector cpu = run_cached_attention("cpu", ceiling, window, sink, h, kv, d, steps, collect_from); + std::vector metal = run_cached_attention("metal", ceiling, window, sink, h, kv, d, steps, collect_from); + if (cpu.size() != metal.size() || cpu.empty()) return false; + for (size_t i = 0; i < cpu.size(); ++i) { + float scale = std::max(1.0f, std::fabs(cpu[i])); + if (std::fabs(cpu[i] - metal[i]) > tolerance * scale) { + std::cout << " mismatch at " << i << " (token " << i / (4 * 32) << "): cpu=" << cpu[i] + << " metal=" << metal[i] << "\n"; + return false; + } + } + return true; +} + +bool parity_sliding_window_cache() { + if (!check_cached_attention(/*ceiling=*/64, /*window=*/8, /*sink=*/2, + /*chunks=*/{5}, /*decode_steps=*/12, 5e-2f)) return false; + return check_cached_attention(/*ceiling=*/64, /*window=*/8, /*sink=*/2, + /*chunks=*/{4, 8}, /*decode_steps=*/6, 5e-2f, /*collect_from=*/2); +} + +bool parity_cache_growth() { + return check_cached_attention(/*ceiling=*/2048, /*window=*/0, /*sink=*/2, + /*chunks=*/{600}, /*decode_steps=*/3, 5e-2f); +} + +} + +int main() { + TestRunner runner("Metal Parity (CPU vs Metal per-op)"); + if (!metal_present()) { +#if defined(__APPLE__) + if (std::getenv("CACTUS_PARITY_ALLOW_SKIP")) { + std::cout << "Metal backend unavailable; skipping parity suite (CACTUS_PARITY_ALLOW_SKIP).\n"; + return 0; + } + std::cout << "FAIL: Metal backend unavailable on Apple hardware " + "(set CACTUS_PARITY_ALLOW_SKIP=1 to skip).\n"; + return 1; +#else + std::cout << "Metal backend unavailable; skipping parity suite.\n"; + return 0; +#endif + } + + runner.run_test("abs", parity_unary(&CactusGraph::abs)); + runner.run_test("scalar_exp", parity_unary(&CactusGraph::scalar_exp)); + runner.run_test("scalar_sqrt", parity_unary(&CactusGraph::scalar_sqrt, 0.1f, 2.0f)); + runner.run_test("scalar_cos", parity_unary(&CactusGraph::scalar_cos)); + runner.run_test("scalar_sin", parity_unary(&CactusGraph::scalar_sin)); + runner.run_test("scalar_log", parity_unary(&CactusGraph::scalar_log, 0.1f, 3.0f)); + runner.run_test("gelu_erf", parity_unary(&CactusGraph::gelu_erf)); + runner.run_test("sigmoid", parity_unary(&CactusGraph::sigmoid)); + runner.run_test("pow", parity_scalar(&CactusGraph::pow, 2.0f, 0.1f, 2.0f)); + runner.run_test("leaky_relu", parity_scalar(&CactusGraph::leaky_relu, 0.1f)); + runner.run_test("scalar_not_equal", parity_scalar(&CactusGraph::scalar_not_equal, 0.0f)); + runner.run_test("not_equal", parity_not_equal()); + runner.run_test("broadcast_binary", parity_broadcast_binary()); + runner.run_test("sum", parity_reduce(&CactusGraph::sum)); + runner.run_test("mean", parity_reduce(&CactusGraph::mean)); + runner.run_test("variance", parity_reduce(&CactusGraph::variance)); + runner.run_test("min", parity_reduce(&CactusGraph::min)); + runner.run_test("max", parity_reduce(&CactusGraph::max)); + runner.run_test("cumsum", parity_cumsum()); + runner.run_test("concat", parity_concat()); + runner.run_test("gather", parity_gather()); + runner.run_test("rope", parity_rope(false)); + runner.run_test("rope_gptj", parity_rope(true)); + runner.run_test("maxpool1d", parity_maxpool1d()); + runner.run_test("bilinear_interpolation", parity_bilinear()); + runner.run_test("conv1d", parity_conv1d()); + runner.run_test("conv1d_k7s3", parity_conv1d_k7s3()); + runner.run_test("conv1d_causal", parity_conv1d_causal()); + runner.run_test("conv1d_same_depthwise_k9", parity_conv1d_dw_k9()); + runner.run_test("conv1d_pointwise", parity_conv1d_pointwise()); + runner.run_test("conv2d_k3s2p1", parity_conv2d(&CactusGraph::conv2d_k3s2p1, false, false)); + runner.run_test("conv2d_k3s1p1", parity_conv2d(&CactusGraph::conv2d_k3s1p1, false, false)); + runner.run_test("conv2d_depthwise_k3s2p1", parity_conv2d(&CactusGraph::conv2d_depthwise_k3s2p1, true, false)); + runner.run_test("conv2d_pointwise_1x1", parity_conv2d(&CactusGraph::conv2d_pointwise_1x1, false, true)); + runner.run_test("batchnorm", parity_batchnorm()); + runner.run_test("groupnorm", parity_groupnorm()); + runner.run_test("softmax", parity_softmax()); + runner.run_test("rms_norm", parity_rms_norm()); + runner.run_test("residual_rms_norm (fused)", parity_residual_rms_norm()); + runner.run_test("elemwise_chain (fused)", parity_elemwise_chain()); + runner.run_test("flash_attention (T%64!=0)", parity_flash_attention()); + runner.run_test("attention_causal", parity_attention_causal()); + runner.run_test("equality_exact", parity_equality_exact()); + runner.run_test("sliding_window_cache (ring wrap)", parity_sliding_window_cache()); + runner.run_test("cache_growth", parity_cache_growth()); + + runner.print_summary(); + return runner.all_passed() ? 0 : 1; +} diff --git a/cactus-kernels/CMakeLists.txt b/cactus-kernels/CMakeLists.txt index 5e1a805d2..20d76dc4f 100644 --- a/cactus-kernels/CMakeLists.txt +++ b/cactus-kernels/CMakeLists.txt @@ -65,6 +65,21 @@ if(APPLE) set(CMAKE_OSX_ARCHITECTURES "arm64") target_compile_definitions(cactus_kernels PUBLIC ACCELERATE_NEW_LAPACK) target_link_libraries(cactus_kernels PUBLIC "-framework Accelerate") + enable_language(OBJCXX) + set(_msl_in ${CMAKE_CURRENT_SOURCE_DIR}/src/cactus_kernels.metal) + set(_msl_out ${CMAKE_CURRENT_BINARY_DIR}/generated/cactus_kernels_msl.h) + add_custom_command( + OUTPUT ${_msl_out} + COMMAND ${CMAKE_COMMAND} -DIN=${_msl_in} -DOUT=${_msl_out} -DDELIM=CACTUS_MSL + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/embed_msl.cmake + DEPENDS ${_msl_in} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/embed_msl.cmake + VERBATIM) + target_sources(cactus_kernels PRIVATE src/metal_backend.mm ${_msl_out}) + target_include_directories(cactus_kernels PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/generated) + set_source_files_properties(src/metal_backend.mm PROPERTIES COMPILE_FLAGS "-fobjc-arc") + target_link_libraries(cactus_kernels PUBLIC "-framework Metal" "-framework Foundation" "-framework MetalPerformanceShaders") +else() + target_sources(cactus_kernels PRIVATE src/metal_backend_stub.cpp) endif() if((BUILD_TESTING OR CACTUS_BUILD_TESTS) AND CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) diff --git a/cactus-kernels/cmake/embed_msl.cmake b/cactus-kernels/cmake/embed_msl.cmake new file mode 100644 index 000000000..500fde994 --- /dev/null +++ b/cactus-kernels/cmake/embed_msl.cmake @@ -0,0 +1,4 @@ +file(READ "${IN}" SRC_HEX HEX) +string(REGEX REPLACE "(..)" "0x\\1," SRC_BYTES "${SRC_HEX}") +file(WRITE "${OUT}" + "static const char kCactusMSL[] = {${SRC_BYTES}0x00};\n") diff --git a/cactus-kernels/metal_backend.h b/cactus-kernels/metal_backend.h new file mode 100644 index 000000000..9505aad48 --- /dev/null +++ b/cactus-kernels/metal_backend.h @@ -0,0 +1,227 @@ +#ifndef CACTUS_METAL_BACKEND_H +#define CACTUS_METAL_BACKEND_H + +#if defined(__APPLE__) +#define CACTUS_HAS_METAL 1 +#else +#define CACTUS_HAS_METAL 0 +#endif + +#include "cactus_kernels.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif +bool cactus_metal_available(); +#ifdef __cplusplus +} +#endif + +void cactus_metal_set_active(bool active); +bool cactus_metal_active_mode(); + +void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, + uint32_t M, __fp16* C); + +void cactus_metal_session_begin(); +void cactus_metal_session_sync(); +void cactus_metal_session_flush(); +void cactus_metal_invalidate_host_wraps(); +void cactus_metal_trim_prefill_cache(); +void cactus_metal_session_end(); + +void* cactus_metal_alloc_shared(size_t bytes); +void* cactus_metal_alloc_pooled(size_t bytes); +void cactus_metal_free_shared(void* contents); + +bool cactus_metal_encode_copy(void* out, const void* in, size_t bytes); +bool cactus_metal_encode_binary(int op_type, void* out, const void* a, const void* b, size_t n); +bool cactus_metal_encode_scalar(int op_type, void* out, const void* in, size_t n, float param); +bool cactus_metal_encode_unary(int op_type, void* out, const void* in, size_t n); +bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, size_t n, float scale); +bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, + size_t rows, size_t dim, float eps); +bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* weight, const void* res, + size_t rows, size_t dim, float eps); +bool cactus_metal_encode_rms_norm_add_rms(void* h_out, void* xn_out, const void* in, const void* w1, + const void* res, const void* w2, + size_t rows, size_t dim, float eps, float out_scale); +bool cactus_metal_encode_rms_norm_add_scale(void* out, const void* in, const void* weight, const void* res, + size_t rows, size_t dim, float eps, float out_scale); +bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3, const void* bias); +bool cactus_metal_encode_softcap(void* out, const void* in, size_t n, float cap); +bool cactus_metal_encode_adjust_logits(void* logits, size_t vocab, + const uint32_t* recent, uint32_t n_recent, + int64_t suppressed, float penalty); +bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_prec, size_t n); +bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W); +bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M); +bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* const* Ws, int B, void* const* codes); +bool cactus_metal_encode_gemv_precoded(void* out, const void* code, const CactusQuantMatrix* W); +bool cactus_metal_encode_transform_gemv(void* out, const void* x, const CactusQuantMatrix* W, const void* osw); +bool cactus_metal_transform_gemv_fits(uint32_t K); +bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, + const CactusQuantMatrix* const* Ws, int B); +bool cactus_metal_encode_swiglu_transform(void* code, const void* gate, const void* up, + const CactusQuantMatrix* W, float scale); +bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W); +bool cactus_metal_encode_rope_pair(void* out, const void* x, const void* c, const void* s, + uint32_t H, uint32_t D); +bool cactus_metal_encode_rope_pair_rms(void* out, const void* x, const void* w, + const void* c, const void* s, + uint32_t H, uint32_t D, float eps); +bool cactus_metal_encode_deltanet_decode(void* out, const void* q, const void* k, const void* v, + const void* g, const void* b, const void* s, + uint32_t B, uint32_t Hq, uint32_t Hv, + uint32_t K, uint32_t V, float scale); +bool cactus_metal_encode_deltanet_prefill(void* out, const void* q, const void* k, const void* v, + const void* g, const void* b, const void* s, + uint32_t B, uint32_t T, uint32_t Hq, uint32_t Hv, + uint32_t K, uint32_t V, float scale); +bool cactus_metal_encode_rms2_add_clip(void* out, const void* a, const void* wa, + const void* b, const void* wb, size_t dim, + float eps_a, float eps_b); +bool cactus_metal_encode_rms_norm_scale(void* out, const void* in, const void* weight, + size_t rows, size_t dim, float eps, float oscale); +bool cactus_metal_encode_softmax_topk(void* probs, void* topk, const void* in, + size_t rows, size_t cols, size_t k, float scale); +bool cactus_metal_encode_topk_rows(void* out, const void* in, size_t rows, size_t cols, size_t k); +bool cactus_metal_moe_cq4_ready(const CactusQuantMatrix* w1_0); +bool cactus_metal_moe_cq4_build(const CactusQuantMatrix* w1s, const CactusQuantMatrix* w3s, + const CactusQuantMatrix* w2s, uint32_t num_experts); +bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void* probs, + const void* topk, const CactusQuantMatrix* w1_0, + uint32_t num_experts, uint32_t top_k, uint32_t tokens, + uint32_t act, uint32_t normalize, float eps, float scaling); +bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* code, + const CactusQuantMatrix* W); +bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W, float scale); +bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const CactusQuantMatrix* W); +bool cactus_metal_encode_embedding_ortho_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M); +bool cactus_metal_encode_embedding_hadamard_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M); +bool cactus_metal_encode_gather_f16(void* out, const void* table, size_t table_bytes, const uint32_t* rows, uint32_t M, uint32_t D); + +bool cactus_metal_encode_attention_i8( + void* out, const void* q, const void* knew, const void* vnew, + const void* kc, const void* vc, const void* ks, const void* vs, + uint32_t num_q_heads, uint32_t num_kv_heads, uint32_t head_dim, uint32_t v_hdim, + uint32_t history_len, uint32_t total_keys, uint32_t kv_start, uint32_t kv_end, + float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes); + +bool cactus_metal_encode_attention_fused_i8( + void* out, const void* q, const void* kraw, const void* vraw, + void* kc, void* vc, void* ks, void* vs, + const void* qw, const void* kw, const void* vw, const void* cs, const void* sn, + uint32_t nqh, uint32_t hd, uint32_t vhd, + uint32_t kv_start, uint32_t kv_end, uint32_t slot, uint32_t has_new, + float eps, float scale, + size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes); + +bool cactus_metal_encode_attention_i8_prefill( + void* out, const void* q, const void* knew, const void* vnew, + const void* kc, const void* vc, const void* ks, const void* vs, + uint32_t num_q_heads, uint32_t num_kv_heads, uint32_t head_dim, uint32_t v_hdim, + uint32_t history_len, uint32_t new_len, uint32_t q_pos0, uint32_t window, uint32_t is_causal, uint32_t M, + float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes, + uint32_t sink, uint32_t ring); + +bool cactus_metal_encode_binary_f32(int op, void* out, const void* a, const void* b, size_t n); +bool cactus_metal_encode_scalar_f32(int op, void* out, const void* in, size_t n, float p); +bool cactus_metal_encode_unary_f32(int op, void* out, const void* in, size_t n); +bool cactus_metal_encode_clamp(void* out, const void* in, size_t n, float lo, float hi, int f32); +bool cactus_metal_encode_glu(void* out, const void* in, size_t split, size_t inner, size_t n_out); +bool cactus_metal_encode_layer_norm(void* out, const void* in, const void* w, const void* b, + size_t rows, size_t dim, float eps); +bool cactus_metal_encode_softmax_rows(void* out, const void* in, size_t rows, size_t cols); +bool cactus_metal_encode_conv1d_k3(void* out, const void* x, const void* w, int w_int8, + const void* w_scales, uint32_t w_gs, + uint32_t Cin, uint32_t L, uint32_t Cout, uint32_t Lout, uint32_t stride); +bool cactus_metal_encode_gemm_f16(void* out, const void* lhs, const void* rhs, + uint32_t M, uint32_t K, uint32_t N, int pretransposed); +bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, const void* v, const void* mask, + uint32_t B, uint32_t T, uint32_t S, uint32_t HQ, uint32_t HKV, uint32_t D, uint32_t DV, + float scale, uint32_t causal, uint32_t pos_off, uint32_t window, float logit_cap, uint32_t mask_mode); +bool cactus_metal_encode_reduce_axis(int op, void* out, const void* in, uint32_t outer, + uint32_t axis_size, uint32_t inner, int f32); +bool cactus_metal_encode_cumsum(void* out, const void* in, uint32_t outer, + uint32_t axis_size, uint32_t inner, int f32); +bool cactus_metal_encode_concat2(void* out, const void* a, const void* b, + uint32_t a_outer, uint32_t b_outer, + uint32_t a_axis, uint32_t b_axis, uint32_t inner); +bool cactus_metal_encode_gather_f32idx(void* out, const void* table, const void* idx, + uint32_t rows, uint32_t D, size_t table_bytes); +bool cactus_metal_encode_rope_full(void* out, const void* in, uint32_t tokens, uint32_t S, + uint32_t H, uint32_t D, uint32_t rot, uint32_t pos0, + float theta, int gptj); +bool cactus_metal_encode_maxpool1d(void* out, const void* in, uint32_t NC, uint32_t L, + uint32_t Lout, uint32_t K, uint32_t stride); +bool cactus_metal_encode_bilinear(void* out, const void* in, uint32_t sh, uint32_t sw, + uint32_t dh, uint32_t dw, uint32_t E, int align); +bool cactus_metal_encode_conv1d_gen(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t Cin, uint32_t L, uint32_t Cout, + uint32_t Lout, uint32_t K, uint32_t stride, int w_ck_co); +bool cactus_metal_encode_conv1d_nlc_dw(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t L, uint32_t C, uint32_t K, + uint32_t dil, uint32_t pad); +bool cactus_metal_encode_conv2d(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t Cin, uint32_t H, uint32_t W, uint32_t Cout, + uint32_t Ho, uint32_t Wo, uint32_t K, uint32_t stride, + uint32_t pad, int dw); +bool cactus_metal_encode_batchnorm(void* out, const void* x, const void* w, const void* b, + const void* rm, const void* rv, uint32_t C, uint32_t inner, + uint32_t total, float eps); +bool cactus_metal_encode_groupnorm(void* out, const void* x, const void* w, const void* b, + uint32_t N, uint32_t C, uint32_t S, uint32_t groups, float eps); +bool cactus_metal_encode_bias_add_rows(void* y, const void* bias, uint32_t C, uint32_t total); +bool cactus_metal_encode_elemwise_chain(void* out, const void* in, const float* steps, + uint32_t nsteps, const void* side0, const void* side1, + const void* side2, const size_t* side_elems, + size_t n, uint32_t flags, uint32_t inner); +bool cactus_metal_encode_rms_norm_add_rows(void* ysum, void* ynorm, const void* x, const void* res, + const void* w, uint32_t rows, uint32_t dim, float eps, + int clipped); +bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, + uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a); +bool cactus_metal_encode_conv1d_dw(void* out, const void* x, const void* w, + uint32_t C, uint32_t L, uint32_t Lout, uint32_t K, uint32_t stride); +bool cactus_metal_encode_transpose2d(void* out, const void* in, uint32_t batch, uint32_t R, uint32_t C); +bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* oshape, + const uint32_t* sstride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes); + +bool cactus_metal_encode_bcast_binary(int op, void* out, const void* a, const void* b, + const uint32_t* oshape, const uint32_t* astride, const uint32_t* bstride, uint32_t ndim, uint32_t total, + size_t a_bytes, size_t b_bytes, size_t out_bytes); + +bool cactus_metal_encode_strided_scatter(void* out, const void* in, const uint32_t* ishape, + const uint32_t* ostride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes); + +bool cactus_metal_encode_kv_append_i8(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, + size_t src_bytes, size_t int8_bytes, size_t scale_bytes); +bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, + uint32_t group_size, size_t src_bytes, size_t int8_bytes, size_t scale_bytes); +bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, + uint32_t group_size, uint32_t M, size_t src_bytes, size_t int8_bytes, size_t scale_bytes); + +bool cactus_metal_encode_kv_append_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, + size_t src_bytes, size_t int8_bytes, size_t scale_bytes); + +bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, + uint32_t sink, uint32_t W, size_t src_bytes, size_t int8_bytes, size_t scale_bytes); + +bool cactus_metal_encode_conv_cache_append(void* out, const void* src, void* ring, + uint32_t hd, uint32_t ws, uint32_t nnew, uint32_t head0, uint32_t count_new, + uint32_t num_rows, int src_f32); + +bool cactus_metal_encode_rel_pos_bias(void* y, const void* q, const void* r, + uint32_t B, uint32_t T, uint32_t H, uint32_t D, uint32_t R, int r_batched, float scale); + +bool cactus_metal_encode_gemv_bias(void* out, const void* x, const void* w, const void* bias, + uint32_t K, uint32_t N, int pretransposed); + +#endif diff --git a/cactus-kernels/src/blas.cpp b/cactus-kernels/src/blas.cpp index 34adab4e9..db58d4e88 100644 --- a/cactus-kernels/src/blas.cpp +++ b/cactus-kernels/src/blas.cpp @@ -331,6 +331,15 @@ void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* outpu size_t axis_size1 = shape1[axis]; size_t axis_size2 = shape2[axis]; + size_t outer_size1 = 1; + size_t outer_size2 = 1; + for (size_t i = 0; i < static_cast(axis); ++i) { + outer_size1 *= shape1[i]; + outer_size2 *= shape2[i]; + } + if (outer_size1 == 0) outer_size1 = 1; + if (outer_size2 == 0) outer_size2 = 1; + size_t input1_stride = axis_size1 * inner_size; size_t input2_stride = axis_size2 * inner_size; size_t output_stride = (axis_size1 + axis_size2) * inner_size; @@ -338,8 +347,8 @@ void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* outpu CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, [&](size_t start, size_t end) { for (size_t outer = start; outer < end; ++outer) { - const __fp16* in1_ptr = input1 + outer * input1_stride; - const __fp16* in2_ptr = input2 + outer * input2_stride; + const __fp16* in1_ptr = input1 + (outer % outer_size1) * input1_stride; + const __fp16* in2_ptr = input2 + (outer % outer_size2) * input2_stride; __fp16* out_ptr = output + outer * output_stride; size_t copy_size1 = axis_size1 * inner_size; @@ -365,6 +374,14 @@ void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_ inner_size *= output_shape[i]; } + std::vector input_outer(num_inputs, 1); + for (size_t input_idx = 0; input_idx < num_inputs; ++input_idx) { + for (size_t i = 0; i < static_cast(axis); ++i) { + input_outer[input_idx] *= input_shapes[input_idx][i]; + } + if (input_outer[input_idx] == 0) input_outer[input_idx] = 1; + } + CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, [&](size_t start, size_t end) { for (size_t outer = start; outer < end; ++outer) { @@ -372,7 +389,8 @@ void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_ size_t offset = 0; for (size_t input_idx = 0; input_idx < num_inputs; ++input_idx) { - const __fp16* in_ptr = inputs[input_idx] + outer * inner_size * input_shapes[input_idx][axis]; + size_t src_outer = outer % input_outer[input_idx]; + const __fp16* in_ptr = inputs[input_idx] + src_outer * inner_size * input_shapes[input_idx][axis]; size_t copy_size = input_shapes[input_idx][axis] * inner_size; std::memcpy(out_ptr + offset, in_ptr, copy_size * sizeof(__fp16)); offset += copy_size; diff --git a/cactus-kernels/src/cactus_kernels.metal b/cactus-kernels/src/cactus_kernels.metal new file mode 100644 index 000000000..01c3b8b4d --- /dev/null +++ b/cactus-kernels/src/cactus_kernels.metal @@ -0,0 +1,3795 @@ +#include +using namespace metal; +#define ROWS 8u + +inline uint cq_idx(device const uchar* packed, uint k, uint bits) { + if (bits == 4u) return (packed[k >> 1] >> ((k & 1u) * 4u)) & 0xFu; + if (bits == 2u) return (packed[k >> 2] >> ((k & 3u) * 2u)) & 0x3u; + uint bit_pos = k * 3u; + uint byte_idx = bit_pos >> 3, bit_idx = bit_pos & 7u; + uint val = (uint)packed[byte_idx] >> bit_idx; + if (bit_idx > 5u) val |= (uint)packed[byte_idx + 1u] << (8u - bit_idx); + return val & 0x7u; +} + +inline float gelu_tanh(float x) { + float c = 0.7978845608028654f * (x + 0.044715f*x*x*x); + return 0.5f * x * (1.0f + precise::tanh(c)); +} + +static inline void cq4_hada128(thread float& x0, thread float& x1, + thread float& x2, thread float& x3, uint lane) { + float a0=x0+x1,a1=x0-x1,a2=x2+x3,a3=x2-x3; + x0=a0+a2; x1=a1+a3; x2=a0-a2; x3=a1-a3; + #pragma clang loop unroll(full) + for (uint d=1u; d<=16u; d<<=1){ + bool hi=(lane&d)!=0u; + float p0=simd_shuffle_xor(x0,d),p1=simd_shuffle_xor(x1,d),p2=simd_shuffle_xor(x2,d),p3=simd_shuffle_xor(x3,d); + x0=hi?p0-x0:x0+p0; x1=hi?p1-x1:x1+p1; x2=hi?p2-x2:x2+p2; x3=hi?p3-x3:x3+p3; + } + float s=rsqrt(128.0f); + x0*=s; x1*=s; x2*=s; x3*=s; +} + +static inline float cq4_dot64(half4 c0, half4 c1, half4 c2, half4 c3, + ushort4 w, threadgroup const float* cb) { + return (float)c0.x*cb[w[0]&0xF] + (float)c0.y*cb[(w[0]>>4)&0xF] + (float)c0.z*cb[(w[0]>>8)&0xF] + (float)c0.w*cb[(w[0]>>12)&0xF] + + (float)c1.x*cb[w[1]&0xF] + (float)c1.y*cb[(w[1]>>4)&0xF] + (float)c1.z*cb[(w[1]>>8)&0xF] + (float)c1.w*cb[(w[1]>>12)&0xF] + + (float)c2.x*cb[w[2]&0xF] + (float)c2.y*cb[(w[2]>>4)&0xF] + (float)c2.z*cb[(w[2]>>8)&0xF] + (float)c2.w*cb[(w[2]>>12)&0xF] + + (float)c3.x*cb[w[3]&0xF] + (float)c3.y*cb[(w[3]>>4)&0xF] + (float)c3.z*cb[(w[3]>>8)&0xF] + (float)c3.w*cb[(w[3]>>12)&0xF]; +} + +static inline device const uchar4* cq4_il_base(device const uchar* packed, uint n, uint ng, uint g, uint pgb) { + return (device const uchar4*)(packed + ((size_t)(n>>2)*ng+g)*4u*(size_t)pgb + (n&3u)*4u); +} + +static inline float cq4_dot64_il(half4 c0, half4 c1, half4 c2, half4 c3, + uchar4 A, uchar4 B, threadgroup const float* cb) { + return (float)c0.x*cb[A.x&0xFu] + (float)c0.y*cb[A.y&0xFu] + (float)c0.z*cb[A.z&0xFu] + (float)c0.w*cb[A.w&0xFu] + + (float)c1.x*cb[A.x>>4] + (float)c1.y*cb[A.y>>4] + (float)c1.z*cb[A.z>>4] + (float)c1.w*cb[A.w>>4] + + (float)c2.x*cb[B.x&0xFu] + (float)c2.y*cb[B.y&0xFu] + (float)c2.z*cb[B.z&0xFu] + (float)c2.w*cb[B.w&0xFu] + + (float)c3.x*cb[B.x>>4] + (float)c3.y*cb[B.y>>4] + (float)c3.z*cb[B.z>>4] + (float)c3.w*cb[B.w>>4]; +} + +static inline float cq4_norm_at(device const half* norms, uint n, uint ng, uint g, uint il) { + return (float)(il != 0u ? norms[(((size_t)(n>>2)*ng+g)<<2) + (n&3u)] + : norms[(size_t)n*ng+g]); +} + +static inline uint cq4_il_nib(device const uchar* packed, uint n, uint ng, uint g, uint pgb, uint e) { + uint v = e>>4, j = e&15u, sub = j>>2, b = j&3u; + device const uchar* panel = packed + ((size_t)(n>>2)*ng+g)*4u*(size_t)pgb; + uchar by = panel[(2u*v + (sub>>1))*16u + (n&3u)*4u + b]; + return (sub&1u) ? (uint)(by>>4) : (uint)(by&0xFu); +} + +static inline uint cq_il_idx(device const uchar* packed, uint n, uint ng, uint g, uint pgb, uint e, uint bits) { + if (bits == 4u) return cq4_il_nib(packed, n, ng, g, pgb, e); + device const uchar* panel = packed + ((size_t)(n>>2)*ng+g)*4u*(size_t)pgb; + uint r = n&3u; + if (bits == 2u) { + uchar by = panel[(e>>4)*16u + ((e&15u)>>2)*4u + r]; + return ((uint)by >> (2u*(e&3u))) & 3u; + } + device const uchar* ch = panel + (e>>2)*6u; + ulong word = (ulong)ch[0] | ((ulong)ch[1]<<8) | ((ulong)ch[2]<<16) + | ((ulong)ch[3]<<24) | ((ulong)ch[4]<<32) | ((ulong)ch[5]<<40); + return (uint)((word >> (r*12u + (e&3u)*3u)) & 7u); +} + +static inline float cq4_dot64_ilw(half4 c0, half4 c1, half4 c2, half4 c3, + uint A, uint B, threadgroup const float* cb) { + return (float)c0.x*cb[A&0xFu] + (float)c0.y*cb[(A>>8)&0xFu] + (float)c0.z*cb[(A>>16)&0xFu] + (float)c0.w*cb[(A>>24)&0xFu] + + (float)c1.x*cb[(A>>4)&0xFu] + (float)c1.y*cb[(A>>12)&0xFu] + (float)c1.z*cb[(A>>20)&0xFu] + (float)c1.w*cb[A>>28] + + (float)c2.x*cb[B&0xFu] + (float)c2.y*cb[(B>>8)&0xFu] + (float)c2.z*cb[(B>>16)&0xFu] + (float)c2.w*cb[(B>>24)&0xFu] + + (float)c3.x*cb[(B>>4)&0xFu] + (float)c3.y*cb[(B>>12)&0xFu] + (float)c3.z*cb[(B>>20)&0xFu] + (float)c3.w*cb[B>>28]; +} + +kernel void cq4_transform( + device const half* x [[buffer(0)]], + device const half* recip [[buffer(1)]], + device const char* lsign [[buffer(2)]], + device const char* rsign [[buffer(3)]], + device const uint* perm [[buffer(4)]], + device half* code [[buffer(5)]], + constant uint& gs [[buffer(6)]], + uint g [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* z [[threadgroup(0)]]) +{ + for (uint k=t; k=N) return; + uint K = num_groups*gs; + float acc=0; + for (uint base=lane*CQ4_VPL; base>4)*8u], B = pp[(off>>4)*8u+4u]; + float p = cq4_dot64_il(cbase[0], cbase[1], cbase[2], cbase[3], A, B, cb); + acc += (float)norms[(((size_t)(n>>2)*num_groups+g)<<2) + (n&3u)]*p; + } else { + device const ushort4* pr=(device const ushort4*)(packed + ((size_t)n*num_groups+g)*pgb + off/2u); + ushort4 w=pr[0]; + float p = 0; + #pragma clang loop unroll(full) + for (uint q=0;q<4;++q){ + half4 c=cbase[q]; + ushort ww = w[q]; + p += (float)c.x*cb[ww&0xF] + (float)c.y*cb[(ww>>4)&0xF] + + (float)c.z*cb[(ww>>8)&0xF] + (float)c.w*cb[(ww>>12)&0xF]; + } + acc += (float)norms[(size_t)n*num_groups+g]*p; + } + } + acc=simd_sum(acc); + if (lane==0) y[n]=(half)acc; +} + +#define CQ4_NR 2u +kernel void cq4_gemv_mr( + device const half* code [[buffer(0)]], + device const uchar* packed [[buffer(1)]], + device const half* codebook [[buffer(2)]], + device const half* norms [[buffer(3)]], + device half* y [[buffer(4)]], + constant uint& gs [[buffer(5)]], + constant uint& num_groups [[buffer(6)]], + constant uint& pgb [[buffer(7)]], + constant uint& N [[buffer(8)]], + constant uint& il [[buffer(9)]], + uint tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]]) +{ + threadgroup float cb[16]; + if (tl<16) cb[tl]=(float)codebook[tl]; + threadgroup_barrier(mem_flags::mem_threadgroup); + uint n0 = (tg*ROWS + sgid)*CQ4_NR; + if (n0>=N) return; + uint K = num_groups*gs; + float acc[CQ4_NR]; + #pragma clang loop unroll(full) + for (uint r=0;r>2)*num_groups+g)*4u*(size_t)pgb + + (n0&3u)*4u + (off>>4)*32u; + uint2 Aw = ((device const uint2*)pan)[0]; + uint2 Bw = ((device const uint2*)(pan+16u))[0]; + #pragma clang loop unroll(full) + for (uint r=0;r>2)*num_groups+g)<<2) + (n0&3u) + r]*p; + } + } else { + #pragma clang loop unroll(full) + for (uint r=0;r>2)*ng+g)*256u + (n0&3u)*4u + (off>>4)*32u; + uint2 Aw = ((device const uint2*)pan)[0]; + uint2 Bw = ((device const uint2*)(pan+16u))[0]; + #pragma clang loop unroll(full) + for (uint r=0u;r<2u;r++){ + float p = cq4_dot64_ilw(c0, c1, c2, c3, r?Aw.y:Aw.x, r?Bw.y:Bw.x, cb); + acc[r] += (float)norms[(((size_t)(n0>>2)*ng+g)<<2) + (n0&3u) + r]*p; + } + } + #pragma clang loop unroll(full) + for (uint r=0u;r<2u;r++){ + float a = simd_sum(acc[r]); + if (lane==0u){ + uint n = n0+r; + if (U.oswi != 0u) { + float gg = gelu_tanh((float)(half)a) * (float)osw[n]; + y[n] = (half)clamp(gg,-65504.0f,65504.0f); + } else { + y[n] = (half)a; + } + } + } +} + +kernel void cq4_swiglu_transform( + device const half* gate [[buffer(0)]], + device const half* up [[buffer(1)]], + device const half* recip [[buffer(2)]], + device const char* lsign [[buffer(3)]], + device const char* rsign [[buffer(4)]], + device const uint* perm [[buffer(5)]], + device half* code [[buffer(6)]], + constant float& scale [[buffer(7)]], + uint g [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + threadgroup float* zmem [[threadgroup(0)]]) +{ + uint b = g*128u + lane*4u; + uint k = lane*4u; + float x0,x1,x2,x3; + { + float g0 = gelu_tanh((float)gate[b+0]) * scale * (float)up[b+0]; + float g1 = gelu_tanh((float)gate[b+1]) * scale * (float)up[b+1]; + float g2 = gelu_tanh((float)gate[b+2]) * scale * (float)up[b+2]; + float g3 = gelu_tanh((float)gate[b+3]) * scale * (float)up[b+3]; + x0=(float)(half)clamp(g0,-65504.0f,65504.0f)*(float)recip[b+0]*(float)lsign[k+0]; + x1=(float)(half)clamp(g1,-65504.0f,65504.0f)*(float)recip[b+1]*(float)lsign[k+1]; + x2=(float)(half)clamp(g2,-65504.0f,65504.0f)*(float)recip[b+2]*(float)lsign[k+2]; + x3=(float)(half)clamp(g3,-65504.0f,65504.0f)*(float)recip[b+3]*(float)lsign[k+3]; + } + cq4_hada128(x0, x1, x2, x3, lane); + zmem[k+0]=x0*(float)rsign[k+0]; zmem[k+1]=x1*(float)rsign[k+1]; + zmem[k+2]=x2*(float)rsign[k+2]; zmem[k+3]=x3*(float)rsign[k+3]; + threadgroup_barrier(mem_flags::mem_threadgroup); + code[b+0]=(half)zmem[perm[k+0]]; code[b+1]=(half)zmem[perm[k+1]]; + code[b+2]=(half)zmem[perm[k+2]]; code[b+3]=(half)zmem[perm[k+3]]; +} + +struct CatU { uint ng, N0, N1, N2; }; +kernel void cq4_gemv_cat( + device const half* code0 [[buffer(0)]], device const half* code1 [[buffer(1)]], device const half* code2 [[buffer(2)]], + device const uchar* packed0[[buffer(3)]], device const uchar* packed1[[buffer(4)]], device const uchar* packed2[[buffer(5)]], + device const half* cbook0 [[buffer(6)]], device const half* cbook1 [[buffer(7)]], device const half* cbook2 [[buffer(8)]], + device const half* norms0 [[buffer(9)]], device const half* norms1 [[buffer(10)]], device const half* norms2 [[buffer(11)]], + device half* y0 [[buffer(12)]], device half* y1 [[buffer(13)]], device half* y2 [[buffer(14)]], + constant CatU& U [[buffer(15)]], + uint tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]]) +{ + uint n0 = tg*16u; + uint b, nbase; + if (n0 < U.N0) { b = 0u; nbase = 0u; } + else if (n0 < U.N0 + U.N1) { b = 1u; nbase = U.N0; } + else { b = 2u; nbase = U.N0 + U.N1; } + device const half* code = (b==0u)?code0 :((b==1u)?code1 :code2 ); + device const uchar* packed= (b==0u)?packed0:((b==1u)?packed1:packed2); + device const half* cbook = (b==0u)?cbook0:((b==1u)?cbook1:cbook2); + device const half* norms = (b==0u)?norms0:((b==1u)?norms1:norms2); + device half* y = (b==0u)?y0 :((b==1u)?y1 :y2 ); + threadgroup float cb[16]; + if (tl<16) cb[tl]=(float)cbook[tl]; + threadgroup_barrier(mem_flags::mem_threadgroup); + uint ng = U.ng; + uint K = ng*128u; + uint nl0 = (n0 - nbase) + sgid*2u; + float acc[2]; acc[0]=0; acc[1]=0; + for (uint base=lane*16u; base>2)*ng+g)*256u + (nl0&3u)*4u + (off>>4)*32u; + uint2 Aw = ((device const uint2*)pan)[0]; + uint2 Bw = ((device const uint2*)(pan+16u))[0]; + #pragma clang loop unroll(full) + for (uint r=0;r<2;r++){ + float p = cq4_dot64_ilw(c0, c1, c2, c3, r?Aw.y:Aw.x, r?Bw.y:Bw.x, cb); + acc[r] += (float)norms[(((size_t)(nl0>>2)*ng+g)<<2) + (nl0&3u) + r]*p; + } + } + #pragma clang loop unroll(full) + for (uint r=0;r<2;r++){ + float a = simd_sum(acc[r]); + if (lane==0) y[nl0+r]=(half)a; + } +} + +kernel void cq_gemv_mr_lowbit( + device const half* code [[buffer(0)]], + device const uchar* packed [[buffer(1)]], + device const half* codebook [[buffer(2)]], + device const half* norms [[buffer(3)]], + device half* y [[buffer(4)]], + constant uint& gs [[buffer(5)]], + constant uint& num_groups [[buffer(6)]], + constant uint& pgb [[buffer(7)]], + constant uint& N [[buffer(8)]], + constant uint& bits [[buffer(9)]], + constant uint& il [[buffer(10)]], + uint tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]]) +{ + threadgroup float cb[8]; + uint cbn = 1u << bits; + if (tl < cbn) cb[tl] = (float)codebook[tl]; + threadgroup_barrier(mem_flags::mem_threadgroup); + uint n0 = (tg*ROWS + sgid)*CQ4_NR; + if (n0 >= N) return; + uint K = num_groups*gs; + float acc[CQ4_NR]; + #pragma clang loop unroll(full) + for (uint r=0;r> 3; + #pragma clang loop unroll(full) + for (uint r=0;r>2)*num_groups+g)*4u*(size_t)pgb + byte_off*4u; + uint rr = n&3u; + w = (uint)pan[rr] | ((uint)pan[4u+rr]<<8) | ((uint)pan[8u+rr]<<16) | ((uint)pan[12u+rr]<<24); + } else { + w = *(device const uint*)(packed + ((size_t)n*num_groups+g)*pgb + byte_off); + } + w0=cb[w&3u]; w1=cb[(w>>2)&3u]; w2=cb[(w>>4)&3u]; w3=cb[(w>>6)&3u]; + w4=cb[(w>>8)&3u]; w5=cb[(w>>10)&3u]; w6=cb[(w>>12)&3u]; w7=cb[(w>>14)&3u]; + w8=cb[(w>>16)&3u]; w9=cb[(w>>18)&3u]; wa=cb[(w>>20)&3u]; wb=cb[(w>>22)&3u]; + wc=cb[(w>>24)&3u]; wd=cb[(w>>26)&3u]; we=cb[(w>>28)&3u]; wf=cb[(w>>30)&3u]; + } else { + ulong v; + if (il != 0u) { + device const uchar* pan = packed + ((size_t)(n>>2)*num_groups+g)*4u*(size_t)pgb + byte_off*4u; + uint rr = n&3u; + v = 0; + #pragma clang loop unroll(full) + for (uint q=0;q<4u;q++){ + device const uchar* ch = pan + q*6u; + ulong word = (ulong)ch[0] | ((ulong)ch[1]<<8) | ((ulong)ch[2]<<16) + | ((ulong)ch[3]<<24) | ((ulong)ch[4]<<32) | ((ulong)ch[5]<<40); + v |= ((word >> (rr*12u)) & 0xFFFul) << (12u*q); + } + } else { + device const ushort* ps = (device const ushort*)(packed + ((size_t)n*num_groups+g)*pgb + byte_off); + v = (ulong)ps[0] | ((ulong)ps[1] << 16) | ((ulong)ps[2] << 32); + } + w0=cb[v&7u]; w1=cb[(v>>3)&7u]; w2=cb[(v>>6)&7u]; w3=cb[(v>>9)&7u]; + w4=cb[(v>>12)&7u]; w5=cb[(v>>15)&7u]; w6=cb[(v>>18)&7u]; w7=cb[(v>>21)&7u]; + w8=cb[(v>>24)&7u]; w9=cb[(v>>27)&7u]; wa=cb[(v>>30)&7u]; wb=cb[(v>>33)&7u]; + wc=cb[(v>>36)&7u]; wd=cb[(v>>39)&7u]; we=cb[(v>>42)&7u]; wf=cb[(v>>45)&7u]; + } + float pacc = (float)c0.x*w0 + (float)c0.y*w1 + (float)c0.z*w2 + (float)c0.w*w3 + + (float)c1.x*w4 + (float)c1.y*w5 + (float)c1.z*w6 + (float)c1.w*w7 + + (float)c2.x*w8 + (float)c2.y*w9 + (float)c2.z*wa + (float)c2.w*wb + + (float)c3.x*wc + (float)c3.y*wd + (float)c3.z*we + (float)c3.w*wf; + acc[r] += cq4_norm_at(norms, n, num_groups, g, il)*pacc; + } + } + #pragma clang loop unroll(full) + for (uint r=0;r= N) return; + float acc = 0.f; + for (uint g = 0u; g < num_groups; ++g) { + uchar b = packed[((size_t)(n>>2)*num_groups + g)*128u + (lane>>2)*16u + (lane&3u)*4u + (n&3u)]; + device const char* a = act_i8 + (size_t)g*128u + lane*4u; + int partial = (int)a[0]*cb[b & 3u] + (int)a[1]*cb[(b >> 2) & 3u] + + (int)a[2]*cb[(b >> 4) & 3u] + (int)a[3]*cb[(b >> 6) & 3u]; + int idot = simd_sum(partial); + if (lane == 0u) { + float ns = (float)norms[(((size_t)(n>>2)*num_groups+g)<<2) + (n&3u)] * cb_scale; + float sc = ns * act_scales[g]; + acc = fma((float)idot, sc, acc); + } + } + if (lane == 0u) y[n] = (half)acc; +} + +kernel void cq4_transform_m( + device const half* x [[buffer(0)]], + device const half* recip [[buffer(1)]], + device const char* lsign [[buffer(2)]], + device const char* rsign [[buffer(3)]], + device const uint* perm [[buffer(4)]], + device half* code [[buffer(5)]], + constant uint& gs [[buffer(6)]], + constant uint& K [[buffer(7)]], + uint pos [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* z [[threadgroup(0)]]) +{ + uint ng = K/gs; + uint g = pos % ng, row = pos / ng; + size_t xb = (size_t)row*K + (size_t)g*gs; + for (uint k=t; k ma[4], mb[2]; + simdgroup_matrix mc[8]; + for (uint i=0;i<8u;i++) mc[i]=make_filled_simdgroup_matrix(0.f); + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint loop_k=0; loop_k>2)*num_groups+g)<<2) + (nrow&3u)]:0.f; + float wv[16]; + if (nrow>4; + uchar4 A = pp[v4*8u], B = pp[v4*8u+4u]; + wv[0]=cb[A.x&0xFu]; wv[1]=cb[A.y&0xFu]; wv[2]=cb[A.z&0xFu]; wv[3]=cb[A.w&0xFu]; + wv[4]=cb[A.x>>4]; wv[5]=cb[A.y>>4]; wv[6]=cb[A.z>>4]; wv[7]=cb[A.w>>4]; + wv[8]=cb[B.x&0xFu]; wv[9]=cb[B.y&0xFu]; wv[10]=cb[B.z&0xFu]; wv[11]=cb[B.w&0xFu]; + wv[12]=cb[B.x>>4]; wv[13]=cb[B.y>>4]; wv[14]=cb[B.z>>4]; wv[15]=cb[B.w>>4]; + } + for (uint i=0;i<16u;i++){ + uint sx=2u*il0+i/8u, sy=(tl/NL0)/8u, lx=(tl/NL0)%8u, ly=i%8u, ib=8u*sx+sy; + float v=0.f; + if (loop_k+16u*il0+i>1u), n_sg = 32u*(sg&1u); + for (uint i=0;i<8u;i++) + simdgroup_store(mc[i], &Cs[(m_sg+8u*(i/4u))*64u + n_sg+8u*(i%4u)], 64u); + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint idx=tl; idx<32u*64u; idx+=128u){ + uint m=idx/64u, n=idx%64u; + if (r1+m ma[4], mb[2]; + simdgroup_matrix mc[8]; + for (uint i=0;i<8u;i++) mc[i]=make_filled_simdgroup_matrix(0.f); + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint loop_k=0; loop_k>1u), n_sg = 32u*(sg&1u); + for (uint i=0;i<8u;i++) + simdgroup_store(mc[i], &Cs[(m_sg+8u*(i/4u))*64u + n_sg+8u*(i%4u)], 64u); + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint idx=tl; idx<32u*64u; idx+=128u){ + uint m=idx/64u, n=idx%64u; + if (r1+m> 5u; + uint j = tg*32u + c; + float acc = 0; + if (j < K) for (uint k=r; k= K) return; + device const half* rrow = rotation + (size_t)j*K; + float acc=0; + for (uint i=lane*4u; i=n) return; + uint row=i/D, d=i%D; + out[i] = table[(size_t)rows[row]*D + d]; +} + +kernel void copy_bytes(device const uchar* in [[buffer(0)]], device uchar* out [[buffer(1)]], + constant uint& n [[buffer(2)]], uint i [[thread_position_in_grid]]) { + if (i= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) { + float av=(float)a[i0+j], bv=(float)b[i0+j], r; + switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; + case 5: r=(av!=bv)?1.0f:0.0f; break; default: r=av+bv; } + if (op==1) r=clamp(r,-65500.0f,65500.0f); + y[i0+j]=(half)r; + } +} + +static inline float erf_approx(float x) { + float sgn = x < 0.0f ? -1.0f : 1.0f; + float ax = fabs(x); + float t = 1.0f / (1.0f + 0.3275911f * ax); + float poly = t * (0.254829592f + t * (-0.284496736f + t * (1.421413741f + + t * (-1.453152027f + t * 1.061405429f)))); + return sgn * (1.0f - poly * precise::exp(-ax * ax)); +} + +static inline float scalar_apply(float v, float p, int op) { + switch (op) { + case 0: return v+p; + case 1: return v-p; + case 3: return v/p; + case 4: return precise::exp(v); + case 5: return precise::sqrt(v); + case 6: return precise::cos(v); + case 7: return precise::sin(v); + case 8: return precise::log(v); + case 9: return fabs(v); + case 10: return precise::pow(v, p); + case 11: return (v != p) ? 1.0f : 0.0f; + case 12: return (v > 0.0f) ? v : v*p; + default: return v*p; + } +} + +kernel void scalar_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], + constant float& p [[buffer(4)]], uint i [[thread_position_in_grid]]) { + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) y[i0+j]=(half)scalar_apply((float)in[i0+j], p, op); +} + +kernel void unary_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], + uint i [[thread_position_in_grid]]) { + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) { + float x=(float)in[i0+j], r; + if (op==0) r=gelu_tanh(x); + else if (op==1) r=precise::tanh(x); + else if (op==2) r=x/(1.0f+precise::exp(-x)); + else if (op==4) r=0.5f*x*(1.0f+erf_approx(x*0.70710678f)); + else if (op==5) r=1.0f/(1.0f+precise::exp(-x)); + else r=max(x,0.0f); + y[i0+j]=(half)r; + } +} + +kernel void swiglu_f16(device const half* gate [[buffer(0)]], device const half* up [[buffer(1)]], + device half* y [[buffer(2)]], constant uint& n [[buffer(3)]], + constant float& scale [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i>=n) return; + float x=(float)gate[i]; + half g1=(half)gelu_tanh(x); + half g2=(half)((float)g1*scale); + y[i]=(half)((float)g2*(float)up[i]); +} + +kernel void rms_norm_add_simd_f16(device const half* x [[buffer(0)]], device const half* res [[buffer(1)]], + device const half* w [[buffer(2)]], device half* ysum [[buffer(3)]], + device half* ynorm [[buffer(4)]], constant uint& dim [[buffer(5)]], + constant float& eps [[buffer(6)]], constant uint& rows [[buffer(7)]], + constant uint& clipped [[buffer(8)]], + uint tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + uint row = tg * 4 + sgid; + if (row >= rows) return; + device const half* xr = x + (size_t)row * dim; + device const half* rr = res + (size_t)row * dim; + device half* os = ysum + (size_t)row * dim; + device half* on = ynorm + (size_t)row * dim; + float partial = 0; + for (uint i = lane; i < dim; i += 32) { + float v = (float)xr[i] + (float)rr[i]; + if (clipped) v = clamp(v, -65500.0f, 65500.0f); + half h = (half)v; + os[i] = h; + float f = (float)h; + partial += f * f; + } + partial += simd_shuffle_xor(partial, 16); + partial += simd_shuffle_xor(partial, 8); + partial += simd_shuffle_xor(partial, 4); + partial += simd_shuffle_xor(partial, 2); + partial += simd_shuffle_xor(partial, 1); + float inv = 1.0f / sqrt(partial / (float)dim + eps); + for (uint i = lane; i < dim; i += 32) on[i] = (half)((float)os[i] * inv * (float)w[i]); +} + +kernel void rms_norm_simd_f16(device const half* in [[buffer(0)]], device const half* w [[buffer(1)]], + device half* y [[buffer(2)]], constant uint& dim [[buffer(3)]], + constant float& eps [[buffer(4)]], constant uint& rows [[buffer(5)]], + uint tg [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + uint row = tg * 4 + sgid; + if (row >= rows) return; + device const half* x = in + (size_t)row * dim; + device half* o = y + (size_t)row * dim; + float partial = 0; + for (uint i = lane * 4; i + 3 < dim; i += 128) { + half4 v4 = *(device const half4*)(x + i); + float4 f = float4(v4); + partial += f.x*f.x + f.y*f.y + f.z*f.z + f.w*f.w; + } + for (uint i = (dim & ~3u) + lane; i < dim; i += 32) { float v = (float)x[i]; partial += v*v; } + partial += simd_shuffle_xor(partial, 16); + partial += simd_shuffle_xor(partial, 8); + partial += simd_shuffle_xor(partial, 4); + partial += simd_shuffle_xor(partial, 2); + partial += simd_shuffle_xor(partial, 1); + float inv = 1.0f / sqrt(partial / (float)dim + eps); + for (uint i = lane * 4; i + 3 < dim; i += 128) { + half4 v4 = *(device const half4*)(x + i); + half4 w4 = *(device const half4*)(w + i); + float4 f = float4(v4) * inv * float4(w4); + *(device half4*)(o + i) = half4(f); + } + for (uint i = (dim & ~3u) + lane; i < dim; i += 32) o[i] = (half)((float)x[i] * inv * (float)w[i]); +} + +kernel void rms_norm_f16(device const half* in [[buffer(0)]], device const half* w [[buffer(1)]], + device half* y [[buffer(2)]], constant uint& dim [[buffer(3)]], + constant float& eps [[buffer(4)]], + uint row [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], + uint nt [[threads_per_threadgroup]], threadgroup float* red [[threadgroup(0)]]) { + device const half* x = in + (size_t)row*dim; + device half* o = y + (size_t)row*dim; + float partial=0; for (uint i=t;i0; s>>=1){ if (t0; s>>=1){ if (t0; s>>=1){ if (t0; s>>=1){ if (t0; s>>=1){ if (t=total) return; + uint rem=i, src=base; + for (int d=int(ndim)-1; d>=0; --d){ uint c=rem%oshape[d]; rem/=oshape[d]; src+=c*sstride[d]; } + out[i]=in[src]; +} + +kernel void transpose2d_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& R [[buffer(2)]], constant uint& C [[buffer(3)]], + uint3 tgid [[threadgroup_position_in_grid]], + uint3 tp3 [[thread_position_in_threadgroup]]) { + uint2 tp = tp3.xy; + threadgroup half tile[32][33]; + uint b = tgid.z; + uint c0 = tgid.x * 32, r0 = tgid.y * 32; + device const half* src = in + (size_t)b * R * C; + device half* dst = y + (size_t)b * R * C; + for (uint dr = tp.y; dr < 32; dr += 8) { + uint r = r0 + dr, c = c0 + tp.x; + tile[dr][tp.x] = (r < R && c < C) ? src[(size_t)r * C + c] : (half)0; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint dr = tp.y; dr < 32; dr += 8) { + uint c = c0 + dr, r = r0 + tp.x; + if (c < C && r < R) dst[(size_t)c * R + r] = tile[tp.x][dr]; + } +} + +kernel void strided_copy_rows_f16(device const half* in [[buffer(0)]], device half* out [[buffer(1)]], + constant uint* oshape [[buffer(2)]], constant uint* sstride [[buffer(3)]], + constant uint& ndim [[buffer(4)]], constant uint& rows [[buffer(5)]], constant uint& base [[buffer(6)]], + constant uint& inner4 [[buffer(7)]], + uint2 gid [[thread_position_in_grid]]) { + uint v = gid.x, row = gid.y; + if (v >= inner4 || row >= rows) return; + uint rem = row, src = base; + for (int d = int(ndim) - 1; d >= 0; --d) { uint c = rem % oshape[d]; rem /= oshape[d]; src += c * sstride[d]; } + device const half4* s4 = (device const half4*)(in + src); + device half4* d4 = (device half4*)(out + (size_t)row * inner4 * 4); + d4[v] = s4[v]; +} + +kernel void kv_append_i8(device const half* src [[buffer(0)]], device char* int8base [[buffer(1)]], + device float* scalebase [[buffer(2)]], constant uint& kv_heads [[buffer(3)]], + constant uint& hdim [[buffer(4)]], constant uint& current_len [[buffer(5)]], constant uint& group_size [[buffer(6)]], + uint gid [[thread_position_in_grid]]) { + uint num_groups = (hdim + group_size - 1)/group_size; + if (gid >= kv_heads*num_groups) return; + uint h = gid / num_groups, g = gid % num_groups; + uint gstart = g*group_size, gcount = min(group_size, hdim - gstart); + device const half* hs = src + (size_t)h*hdim + gstart; + float maxabs = 0; + for (uint k=0;k= remaining*per) return; + uint s=gid/per, hg=gid%per, h=hg/num_groups, g=hg%num_groups; + uint gstart=g*group_size, gcount=min(group_size, hdim-gstart); + uint i8s=kv_heads*hdim, scs=kv_heads*num_groups, srcseq=shift_src+s; + device const char* sp=int8base+(size_t)srcseq*i8s+(size_t)h*hdim+gstart; + device char* dp=scr_i8+(size_t)s*i8s+(size_t)h*hdim+gstart; + for (uint k=0;k= (remaining+1)*per) return; + uint s=gid/per, hg=gid%per, h=hg/num_groups, g=hg%num_groups; + uint gstart=g*group_size, gcount=min(group_size, hdim-gstart); + uint i8s=kv_heads*hdim, scs=kv_heads*num_groups, dstseq=keep_sink+s; + device char* dp=int8base+(size_t)dstseq*i8s+(size_t)h*hdim+gstart; + if (s < remaining) { + device const char* sp=scr_i8+(size_t)s*i8s+(size_t)h*hdim+gstart; + for (uint k=0;k= (remaining+M)*per) return; + uint s=gid/per, hg=gid%per, h=hg/num_groups, g=hg%num_groups; + uint gstart=g*group_size, gcount=min(group_size, hdim-gstart); + uint i8s=kv_heads*hdim, scs=kv_heads*num_groups, dstseq=keep_sink+s; + device char* dp=int8base+(size_t)dstseq*i8s+(size_t)h*hdim+gstart; + if (s < remaining) { + device const char* sp=scr_i8+(size_t)s*i8s+(size_t)h*hdim+gstart; + for (uint k=0;k=total) return; + uint rem=i, dst=base; + for (int d=int(ndim)-1; d>=0; --d){ uint c=rem%ishape[d]; rem/=ishape[d]; dst+=c*ostride[d]; } + out[dst]=in[i]; +} + +kernel void strided_scatter_rows_f16(device const half* in [[buffer(0)]], device half* out [[buffer(1)]], + constant uint* ishape [[buffer(2)]], constant uint* ostride [[buffer(3)]], + constant uint& ndim [[buffer(4)]], constant uint& rows [[buffer(5)]], constant uint& base [[buffer(6)]], + constant uint& inner4 [[buffer(7)]], + uint2 gid [[thread_position_in_grid]]) { + uint v = gid.x, row = gid.y; + if (v >= inner4 || row >= rows) return; + uint rem = row, dst = base; + for (int d = int(ndim) - 1; d >= 0; --d) { uint c = rem % ishape[d]; rem /= ishape[d]; dst += c * ostride[d]; } + device const half4* s4 = (device const half4*)(in + (size_t)row * inner4 * 4); + device half4* d4 = (device half4*)(out + dst); + d4[v] = s4[v]; +} + +kernel void binary_f32(device const float* a [[buffer(0)]], device const float* b [[buffer(1)]], + device float* y [[buffer(2)]], constant uint& n [[buffer(3)]], + constant int& op [[buffer(4)]], uint i [[thread_position_in_grid]]) { + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) { + float av=a[i0+j], bv=b[i0+j], r; + switch(op){ case 2: r=av-bv; break; case 3: r=av*bv; break; case 4: r=av/bv; break; + case 5: r=(av!=bv)?1.0f:0.0f; break; default: r=av+bv; } + if (op==1) r=clamp(r,-65500.0f,65500.0f); + y[i0+j]=r; + } +} +kernel void scalar_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], + constant float& p [[buffer(4)]], uint i [[thread_position_in_grid]]) { + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) y[i0+j]=scalar_apply(in[i0+j], p, op); +} +kernel void unary_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant int& op [[buffer(3)]], + uint i [[thread_position_in_grid]]) { + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) { + float x=in[i0+j], r; + if (op==0) r=gelu_tanh(x); + else if (op==1) r=precise::tanh(x); + else if (op==2) r=x/(1.0f+precise::exp(-x)); + else if (op==4) r=0.5f*x*(1.0f+erf_approx(x*0.70710678f)); + else if (op==5) r=1.0f/(1.0f+precise::exp(-x)); + else r=max(x,0.0f); + y[i0+j]=r; + } +} + +kernel void clamp_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant float& lo [[buffer(3)]], + constant float& hi [[buffer(4)]], uint i [[thread_position_in_grid]]) { + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) y[i0+j]=(half)clamp((float)in[i0+j], lo, hi); +} + +kernel void clamp_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant float& lo [[buffer(3)]], + constant float& hi [[buffer(4)]], uint i [[thread_position_in_grid]]) { + uint i0 = i * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + for (uint j = 0; j < cnt; ++j) y[i0+j]=clamp(in[i0+j], lo, hi); +} +kernel void glu_f16(device const half* x [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& split [[buffer(2)]], constant uint& inner [[buffer(3)]], + constant uint& n [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i>=n) return; + uint per_outer = split*inner; + uint o = i/per_outer, r = i%per_outer; + size_t base = (size_t)o*2u*per_outer; + float a = (float)x[base + r]; + float g = (float)x[base + per_outer + r]; + y[i] = (half)(a / (1.0f + precise::exp(-g))); +} + +kernel void layer_norm_f16(device const half* in [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* b [[buffer(2)]], device half* y [[buffer(3)]], + constant uint& dim [[buffer(4)]], constant float& eps [[buffer(5)]], + constant uint& has_bias [[buffer(6)]], + uint row [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], + uint nt [[threads_per_threadgroup]], threadgroup float* red [[threadgroup(0)]]) { + device const half* x = in + (size_t)row*dim; + device half* o = y + (size_t)row*dim; + float s=0, ss=0; + for (uint i=t;i0; st>>=1){ if (t0; s>>=1){ if (t0; s>>=1){ if (t0.0f ? 1.0f/red[0] : 0.0f; + for (uint i=t;i=Lout) return; + int center = (int)(gid.x*stride); + float acc=0; + device const half* wr = w + (size_t)gid.y*Cin*3u; + for (uint c=0;c0) ? (float)xc[center-1] : 0.0f; + float x1 = (float)xc[center]; + float x2 = (center+1<(int)L) ? (float)xc[center+1] : 0.0f; + acc = fma(x0,(float)wr[c*3u],acc); + acc = fma(x1,(float)wr[c*3u+1u],acc); + acc = fma(x2,(float)wr[c*3u+2u],acc); + } + y[(size_t)gid.y*Lout + gid.x] = (half)acc; +} + +struct AttnFU { uint T, S, HQ, HKV, D, DV; float scale; uint causal, pos_off, window; float logit_cap; uint mask_mode; }; + +static inline float attn_apply_mask(float score, device const half* mask, constant AttnFU& U, + uint b, uint h, uint t, uint i) { + if (U.mask_mode != 0u) { + size_t mi = (U.mask_mode>=3u) ? (((size_t)b*U.HQ + h)*U.T + t)*U.S + i + : ((size_t)b*U.T + t)*U.S + i; + float mv = (float)mask[mi]; + if (U.mask_mode==1u || U.mask_mode==3u) { score = isfinite(mv) ? score+mv : -INFINITY; } + else if (mv == 0.0f) score = -INFINITY; + } + if (U.logit_cap > 0.0f && score > -INFINITY) score = U.logit_cap * precise::tanh(score / U.logit_cap); + return score; +} + +kernel void attn_f16(device const half* q [[buffer(0)]], device const half* k [[buffer(1)]], + device const half* v [[buffer(2)]], device half* o [[buffer(3)]], + device const half* mask [[buffer(4)]], constant AttnFU& U [[buffer(5)]], + uint3 tg [[threadgroup_position_in_grid]], uint l [[thread_index_in_threadgroup]]) { + const uint QR = 4u; + uint t0 = tg.x * QR, h = tg.y, b = tg.z; + uint kvh = h / (U.HQ / U.HKV); + device const half* kb = k + (size_t)b*U.S*U.HKV*U.D + (size_t)kvh*U.D; + device const half* vb = v + (size_t)b*U.S*U.HKV*U.DV + (size_t)kvh*U.DV; + uint d4 = U.D/4u, dv4 = U.DV/4u; + threadgroup half4 qs[4u*32u]; + uint nq = min(QR, U.T - t0); + for (uint r=0;r0u) { uint aq0 = U.pos_off + t0; kstart = (aq0>U.window) ? aq0-U.window : 0u; } + size_t kstride = (size_t)U.HKV*U.D, vstride = (size_t)U.HKV*U.DV; + for (uint kv0=kstart; kv01u){ float4 q1 = float4(qs[32u+d]); + p.y = fma(kv4.x,q1.x,p.y); p.y = fma(kv4.y,q1.y,p.y); p.y = fma(kv4.z,q1.z,p.y); p.y = fma(kv4.w,q1.w,p.y); } + if (nq>2u){ float4 q2 = float4(qs[64u+d]); + p.z = fma(kv4.x,q2.x,p.z); p.z = fma(kv4.y,q2.y,p.z); p.z = fma(kv4.z,q2.z,p.z); p.z = fma(kv4.w,q2.w,p.z); } + if (nq>3u){ float4 q3 = float4(qs[96u+d]); + p.w = fma(kv4.x,q3.x,p.w); p.w = fma(kv4.y,q3.y,p.w); p.w = fma(kv4.z,q3.z,p.w); p.w = fma(kv4.w,q3.w,p.w); } + } + sc = p * U.scale; + for (uint r=0;r<4u;++r) { + if (r >= nq) { sc[r] = -INFINITY; continue; } + uint t = t0 + r, aq = U.pos_off + t; + if (U.causal && i > aq) { sc[r] = -INFINITY; continue; } + if (U.window>0u && i < aq && (aq - i) > U.window) { sc[r] = -INFINITY; continue; } + sc[r] = attn_apply_mask(sc[r], mask, U, b, h, t, i); + } + } + float4 wgt = float4(0.0f); + bool anyrow = false; + for (uint r=0;r<4u;++r) { + float bm = simd_max(sc[r]); + if (bm > -INFINITY) { + anyrow = true; + if (bm > m[r]) { + float corr = exp(m[r] - bm); + sum[r] *= corr; acc[r][0] *= corr; acc[r][1] *= corr; + m[r] = bm; + } + wgt[r] = (sc[r] > -INFINITY) ? exp(sc[r] - m[r]) : 0.0f; + sum[r] += simd_sum(wgt[r]); + } + } + if (!anyrow) continue; + uint nk = min(32u, kend_all - kv0); + uint half_id = l >> 4u, dl = l & 15u; + for (uint j2=0; j20.0f ? 1.0f/sum[r] : 0.0f; + device half4* orow = (device half4*)(o + ((size_t)b*U.T + t0 + r)*U.HQ*U.DV + (size_t)h*U.DV); + uint ci = 0u; + for (uint dd=dl; dd0u) { uint aq0 = U.pos_off + t0; kstart = (aq0>U.window) ? aq0-U.window : 0u; } + size_t kstride = (size_t)U.HKV*U.D, vstride = (size_t)U.HKV*U.DV; + for (uint kv0=kstart; kv0= nq) { sc[r] = -INFINITY; continue; } + sc[r] = p[r] * U.scale; + uint t = t0 + r, aq = U.pos_off + t; + if (U.causal && i > aq) { sc[r] = -INFINITY; continue; } + if (U.window>0u && i < aq && (aq - i) > U.window) { sc[r] = -INFINITY; continue; } + sc[r] = attn_apply_mask(sc[r], mask, U, b, h, t, i); + } + } + float wgt[8]; + bool anyrow = false; + for (uint r=0;r<8u;++r) { + float bm = simd_max(sc[r]); + if (bm > -INFINITY) { + anyrow = true; + if (bm > m[r]) { + float corr = exp(m[r] - bm); + sum[r] *= corr; acc[r] *= corr; + m[r] = bm; + } + wgt[r] = (sc[r] > -INFINITY) ? exp(sc[r] - m[r]) : 0.0f; + sum[r] += simd_sum(wgt[r]); + } else wgt[r] = 0.0f; + } + if (!anyrow) continue; + uint nk = min(32u, kend_all - kv0); + uint half_id = l >> 4u, dl = l & 15u; + for (uint j2=0; j20.0f ? 1.0f/sum[r] : 0.0f; + device half4* orow = (device half4*)(o + ((size_t)b*U.T + t0 + r)*U.HQ*U.DV + (size_t)h*U.DV); + float4 vv; + vv[0] = acc[r][0] + simd_shuffle_xor(acc[r][0], (ushort)16); + vv[1] = acc[r][1] + simd_shuffle_xor(acc[r][1], (ushort)16); + vv[2] = acc[r][2] + simd_shuffle_xor(acc[r][2], (ushort)16); + vv[3] = acc[r][3] + simd_shuffle_xor(acc[r][3], (ushort)16); + if (l < 16u && dl < dv4) orow[dl] = half4(vv*inv); + } +} + +kernel void attn_flash_f16(device const half* q [[buffer(0)]], device const half* k [[buffer(1)]], + device const half* v [[buffer(2)]], device half* y [[buffer(3)]], + device const half* mask [[buffer(4)]], + constant uint& T [[buffer(5)]], constant uint& S [[buffer(6)]], + constant uint& H [[buffer(7)]], constant float& scale [[buffer(8)]], + constant uint& mask_mode [[buffer(9)]], + uint2 tgid [[threadgroup_position_in_grid]], + uint tid [[thread_index_in_threadgroup]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + constexpr uint BQ = 64, BK = 32, D = 64, NT = 256; + threadgroup half KVs[2 * BK * D]; + threadgroup half* Ks = KVs; + threadgroup half* Vs = KVs + BK * D; + threadgroup float Ss[8][8 * BK]; + threadgroup half Ps[8][8 * BK]; + threadgroup float Dg[8][64]; + threadgroup float Ms[8][8]; + threadgroup float Ls[8][8]; + threadgroup uint Rf[8]; + + uint qb = tgid.x, h = tgid.y; + uint q0 = qb * BQ; + uint row0 = q0 + sgid * 8; + bool interior = q0 + BQ <= T; + + for (uint x = tid; x < 8 * 64; x += NT) Dg[x / 64][x % 64] = 0; + if (tid < 64) { Ms[tid / 8][tid % 8] = -INFINITY; Ls[tid / 8][tid % 8] = 0; } + + simdgroup_half8x8 qf[8]; + if (interior) { + for (uint d = 0; d < 8; ++d) + simdgroup_load(qf[d], q + ((size_t)row0 * H + h) * D + d * 8, (size_t)H * D); + } else { + threadgroup half* Qe = KVs; + for (uint x = tid; x < BQ * D; x += NT) { + uint r = x / D, d = x % D; + uint t = q0 + r; + Qe[x] = (t < T) ? q[((size_t)t * H + h) * D + d] : (half)0; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint d = 0; d < 8; ++d) + simdgroup_load(qf[d], Qe + (size_t)sgid * 8 * D + d * 8, D); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + simdgroup_float8x8 of[8]; + for (uint d = 0; d < 8; ++d) of[d] = make_filled_simdgroup_matrix(0.0f); + + for (uint kb = 0; kb < S; kb += BK) { + for (uint x = tid; x < BK * D / 4; x += NT) { + uint r = (x * 4) / D, d = (x * 4) % D; + uint sidx = kb + r; + half4 kv4 = half4(0.0h), vv4 = half4(0.0h); + if (sidx < S) { + kv4 = *(device const half4*)(k + ((size_t)sidx * H + h) * D + d); + vv4 = *(device const half4*)(v + ((size_t)sidx * H + h) * D + d); + } + *(threadgroup half4*)(Ks + r * D + d) = kv4; + *(threadgroup half4*)(Vs + r * D + d) = vv4; + } + if (lane == 0) Rf[sgid] = 0; + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint j = 0; j < 4; ++j) { + simdgroup_float8x8 sf = make_filled_simdgroup_matrix(0.0f); + for (uint d = 0; d < 8; ++d) { + simdgroup_half8x8 kt; + simdgroup_load(kt, Ks + (size_t)j * 8 * D + d * 8, D, 0, true); + simdgroup_multiply_accumulate(sf, qf[d], kt, sf); + } + simdgroup_store(sf, Ss[sgid] + j * 8, BK); + } + simdgroup_barrier(mem_flags::mem_threadgroup); + + uint r = lane / 4, c0 = (lane % 4) * 8; + uint t = row0 + r; + float sv[8]; + float mloc = -INFINITY; + for (uint c = 0; c < 8; ++c) { + uint sidx = kb + c0 + c; + float val = Ss[sgid][r * BK + c0 + c] * scale; + bool dead = sidx >= S || t >= T; + if (!dead && mask_mode == 2u && mask[(size_t)t * S + sidx] == (half)0.0h) dead = true; + sv[c] = dead ? -INFINITY : val; + mloc = max(mloc, sv[c]); + } + mloc = max(mloc, simd_shuffle_xor(mloc, 1)); + mloc = max(mloc, simd_shuffle_xor(mloc, 2)); + float mold = Ms[sgid][r]; + float mnew = max(mold, mloc); + float sloc = 0; + for (uint c = 0; c < 8; ++c) { + float pv = (mnew == -INFINITY || sv[c] == -INFINITY) ? 0.0f : fast::exp(sv[c] - mnew); + Ps[sgid][r * BK + c0 + c] = (half)pv; + sloc += pv; + } + sloc += simd_shuffle_xor(sloc, 1); + sloc += simd_shuffle_xor(sloc, 2); + float alpha = (mold == -INFINITY) ? 0.0f : fast::exp(mold - mnew); + bool first = mold == -INFINITY; + if ((lane % 4) == 0) { + Ms[sgid][r] = mnew; + Ls[sgid][r] = Ls[sgid][r] * alpha + sloc; + if (mnew != mold && !first) Rf[sgid] = 1; + Dg[sgid][r * 8 + r] = alpha; + } + simdgroup_barrier(mem_flags::mem_threadgroup); + + bool rescale = Rf[sgid] != 0 || kb == 0; + simdgroup_half8x8 pf[4]; + for (uint j = 0; j < 4; ++j) + simdgroup_load(pf[j], Ps[sgid] + j * 8, BK); + if (rescale) { + simdgroup_float8x8 dgf; + simdgroup_load(dgf, Dg[sgid], 8); + for (uint d = 0; d < 8; ++d) { + simdgroup_float8x8 acc; + simdgroup_multiply(acc, dgf, of[d]); + for (uint j = 0; j < 4; ++j) { + simdgroup_half8x8 vf; + simdgroup_load(vf, Vs + (size_t)j * 8 * D + d * 8, D); + simdgroup_multiply_accumulate(acc, pf[j], vf, acc); + } + of[d] = acc; + } + } else { + for (uint d = 0; d < 8; ++d) { + for (uint j = 0; j < 4; ++j) { + simdgroup_half8x8 vf; + simdgroup_load(vf, Vs + (size_t)j * 8 * D + d * 8, D); + simdgroup_multiply_accumulate(of[d], pf[j], vf, of[d]); + } + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + { + uint r = lane / 4; + if ((lane % 4) == 0) { + float l = Ls[sgid][r]; + Dg[sgid][r * 8 + r] = l > 0.0f ? 1.0f / l : 0.0f; + } + simdgroup_barrier(mem_flags::mem_threadgroup); + simdgroup_float8x8 dgf; + simdgroup_load(dgf, Dg[sgid], 8); + if (interior) { + for (uint d = 0; d < 8; ++d) { + simdgroup_multiply(of[d], dgf, of[d]); + simdgroup_store(of[d], Ss[sgid], 8); + simdgroup_barrier(mem_flags::mem_threadgroup); + for (uint e = lane; e < 64; e += 32) { + uint r2 = e / 8, cc = e % 8; + y[((size_t)(row0 + r2) * H + h) * D + d * 8 + cc] = (half)Ss[sgid][e]; + } + simdgroup_barrier(mem_flags::mem_threadgroup); + } + } else { + threadgroup half* Oe = KVs; + for (uint d = 0; d < 8; ++d) { + simdgroup_multiply(of[d], dgf, of[d]); + simdgroup_store(of[d], Ss[sgid], 8); + simdgroup_barrier(mem_flags::mem_threadgroup); + for (uint e = lane; e < 64; e += 32) { + uint r2 = e / 8, cc = e % 8; + Oe[(size_t)sgid * 8 * D + r2 * D + d * 8 + cc] = (half)Ss[sgid][e]; + } + simdgroup_barrier(mem_flags::mem_threadgroup); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint x = tid; x < BQ * D; x += NT) { + uint r2 = x / D, d = x % D; + uint t = q0 + r2; + if (t < T) y[((size_t)t * H + h) * D + d] = Oe[x]; + } + } + } +} + +kernel void attn_maskfill_f16(device const half* mask [[buffer(0)]], device half* out [[buffer(1)]], + constant uint& ts [[buffer(2)]], + uint2 gid [[thread_position_in_grid]]) { + uint i = gid.x, h = gid.y; + if (i >= ts) return; + out[(size_t)h*ts + i] = (mask[i] == (half)0.0f) ? (half)(-30000.0f) : (half)0.0f; +} + +kernel void gemm_batch_f32a(device const float* a [[buffer(0)]], device const half* b [[buffer(1)]], + device uchar* y [[buffer(2)]], + constant uint& M [[buffer(3)]], constant uint& K [[buffer(4)]], + constant uint& N [[buffer(5)]], constant uint& f32out [[buffer(6)]], + uint2 gid [[thread_position_in_grid]]) { + uint mn = gid.x, bidx = gid.y; + if (mn >= M*N) return; + uint m = mn / N, nn = mn % N; + device const float* ar = a + (size_t)bidx*M*K + (size_t)m*K; + device const half* br = b + (size_t)bidx*K*N + nn; + float acc = 0; + for (uint k = 0; k < K; ++k) acc = fma(ar[k], (float)br[(size_t)k*N], acc); + size_t oi = (size_t)bidx*M*N + mn; + if (f32out != 0u) ((device float*)y)[oi] = acc; + else ((device half*)y)[oi] = (half)acc; +} + +kernel void gemm_batch_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], + device uchar* y [[buffer(2)]], + constant uint& M [[buffer(3)]], constant uint& K [[buffer(4)]], + constant uint& N [[buffer(5)]], constant uint& f32out [[buffer(6)]], + uint2 gid [[thread_position_in_grid]]) { + uint mn = gid.x, bidx = gid.y; + if (mn >= M*N) return; + uint m = mn / N, nn = mn % N; + device const half* ar = a + (size_t)bidx*M*K + (size_t)m*K; + device const half* br = b + (size_t)bidx*K*N + nn; + float acc = 0; + for (uint k = 0; k < K; ++k) acc = fma((float)ar[k], (float)br[(size_t)k*N], acc); + size_t oi = (size_t)bidx*M*N + mn; + if (f32out != 0u) ((device float*)y)[oi] = acc; + else ((device half*)y)[oi] = (half)acc; +} + +kernel void conv1d_dw_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device half* y [[buffer(2)]], + constant uint& L [[buffer(3)]], constant uint& Lout [[buffer(4)]], + constant uint& Kk [[buffer(5)]], constant uint& stride [[buffer(6)]], + uint2 gid [[thread_position_in_grid]]) { + uint l = gid.x, c = gid.y; + if (l >= Lout) return; + device const half* xc = x + (size_t)c*L + (size_t)l*stride; + device const half* wc = w + (size_t)c*Kk; + float acc = 0; + for (uint k = 0; k < Kk; ++k) acc = fma((float)xc[k], (float)wc[k], acc); + y[(size_t)c*Lout + l] = (half)acc; +} + +kernel void reduce_axis_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& axis_size [[buffer(2)]], constant uint& inner [[buffer(3)]], + constant uint& n [[buffer(4)]], constant int& op [[buffer(5)]], + uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint outer = i / inner, in_i = i % inner; + size_t base = (size_t)outer * axis_size * inner + in_i; + float r; + if (op <= 2) { + float sum = 0; + for (uint a = 0; a < axis_size; ++a) sum += (float)in[base + (size_t)a*inner]; + float mean = sum / (float)max(axis_size, 1u); + if (op == 0) r = sum; + else if (op == 1) r = mean; + else { + float var = 0; + for (uint a = 0; a < axis_size; ++a) { float d = (float)in[base + (size_t)a*inner] - mean; var += d*d; } + r = var / (float)max(axis_size, 1u); + } + } else { + r = axis_size ? (float)in[base] : 0.0f; + for (uint a = 1; a < axis_size; ++a) { + float v = (float)in[base + (size_t)a*inner]; + r = (op == 3) ? min(r, v) : max(r, v); + } + } + y[i] = (half)r; +} + +kernel void reduce_axis_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], + constant uint& axis_size [[buffer(2)]], constant uint& inner [[buffer(3)]], + constant uint& n [[buffer(4)]], constant int& op [[buffer(5)]], + uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint outer = i / inner, in_i = i % inner; + size_t base = (size_t)outer * axis_size * inner + in_i; + float r; + if (op <= 2) { + float sum = 0; + for (uint a = 0; a < axis_size; ++a) sum += in[base + (size_t)a*inner]; + float mean = sum / (float)max(axis_size, 1u); + if (op == 0) r = sum; + else if (op == 1) r = mean; + else { + float var = 0; + for (uint a = 0; a < axis_size; ++a) { float d = in[base + (size_t)a*inner] - mean; var += d*d; } + r = var / (float)max(axis_size, 1u); + } + } else { + r = axis_size ? in[base] : 0.0f; + for (uint a = 1; a < axis_size; ++a) { + float v = in[base + (size_t)a*inner]; + r = (op == 3) ? min(r, v) : max(r, v); + } + } + y[i] = r; +} + +kernel void cumsum_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& axis_size [[buffer(2)]], constant uint& inner [[buffer(3)]], + constant uint& n [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint outer = i / inner, in_i = i % inner; + size_t base = (size_t)outer * axis_size * inner + in_i; + float run = 0; + for (uint a = 0; a < axis_size; ++a) { + run += (float)in[base + (size_t)a*inner]; + y[base + (size_t)a*inner] = (half)run; + } +} + +kernel void cumsum_f32(device const float* in [[buffer(0)]], device float* y [[buffer(1)]], + constant uint& axis_size [[buffer(2)]], constant uint& inner [[buffer(3)]], + constant uint& n [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint outer = i / inner, in_i = i % inner; + size_t base = (size_t)outer * axis_size * inner + in_i; + float run = 0; + for (uint a = 0; a < axis_size; ++a) { + run += in[base + (size_t)a*inner]; + y[base + (size_t)a*inner] = run; + } +} + +kernel void concat2_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], + device half* y [[buffer(2)]], constant uint& a_axis [[buffer(3)]], + constant uint& b_axis [[buffer(4)]], constant uint& inner [[buffer(5)]], + constant uint& n [[buffer(6)]], constant uint& a_outer [[buffer(7)]], + constant uint& b_outer [[buffer(8)]], uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint in_i = i % inner, rest = i / inner; + uint ab = a_axis + b_axis; + uint ax = rest % ab, outer = rest / ab; + y[i] = (ax < a_axis) ? a[((size_t)(outer % a_outer)*a_axis + ax)*inner + in_i] + : b[((size_t)(outer % b_outer)*b_axis + (ax - a_axis))*inner + in_i]; +} + +kernel void gather_f32idx_f16(device const half* table [[buffer(0)]], device const float* rows [[buffer(1)]], + device half* out [[buffer(2)]], constant uint& D [[buffer(3)]], + constant uint& n [[buffer(4)]], uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint row = i / D, d = i % D; + out[i] = table[(size_t)(uint)rows[row]*D + d]; +} + +kernel void rope_full_f16(device const half* x [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& S [[buffer(2)]], constant uint& H [[buffer(3)]], + constant uint& D [[buffer(4)]], constant uint& rot [[buffer(5)]], + constant uint& pos0 [[buffer(6)]], constant float& theta [[buffer(7)]], + constant uint& gptj [[buffer(8)]], + uint2 gid [[thread_position_in_grid]]) { + uint idx = gid.x, tok = gid.y; + uint hr = rot / 2; + if (idx >= hr) { + uint d = rot + (idx - hr); + if (d >= D) return; + size_t off = (size_t)tok * D + d; + y[off] = x[off]; + return; + } + uint seq = (tok / H) % S; + float freq = 1.0f / precise::pow(theta, (2.0f * idx) / (float)rot); + float angle = (float)(pos0 + seq) * freq; + float c = (float)(half)precise::cos(angle); + float sn = (float)(half)precise::sin(angle); + uint d0 = gptj ? 2u * idx : idx; + uint d1 = gptj ? 2u * idx + 1u : idx + hr; + size_t off0 = (size_t)tok * D + d0, off1 = (size_t)tok * D + d1; + float v0 = (float)x[off0], v1 = (float)x[off1]; + y[off0] = (half)(v0*c - v1*sn); + y[off1] = (half)(v1*c + v0*sn); +} + +kernel void maxpool1d_f16(device const half* x [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& L [[buffer(2)]], constant uint& Lout [[buffer(3)]], + constant uint& K [[buffer(4)]], constant uint& stride [[buffer(5)]], + uint2 gid [[thread_position_in_grid]]) { + uint lo = gid.x, nc = gid.y; + if (lo >= Lout) return; + size_t base = (size_t)nc * L + (size_t)lo * stride; + float r = -INFINITY; + for (uint k = 0; k < K && lo * stride + k < L; ++k) r = max(r, (float)x[base + k]); + y[(size_t)nc * Lout + lo] = (half)r; +} + +kernel void bilinear_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& sh [[buffer(2)]], constant uint& sw [[buffer(3)]], + constant uint& dh [[buffer(4)]], constant uint& dw [[buffer(5)]], + constant uint& E [[buffer(6)]], constant uint& align [[buffer(7)]], + uint2 gid [[thread_position_in_grid]]) { + uint e = gid.x, pix = gid.y; + if (e >= E || pix >= dh * dw) return; + uint dy = pix / dw, dx = pix % dw; + float syf, sxf; + if (align) { + float sc_h = (sh > 1 && dh > 1) ? (float)(sh - 1) / (float)(dh - 1) : 0.0f; + float sc_w = (sw > 1 && dw > 1) ? (float)(sw - 1) / (float)(dw - 1) : 0.0f; + syf = dy * sc_h; sxf = dx * sc_w; + } else { + float sc_h = dh ? (float)sh / (float)dh : 0.0f; + float sc_w = dw ? (float)sw / (float)dw : 0.0f; + syf = clamp((dy + 0.5f) * sc_h - 0.5f, 0.0f, (float)sh - 1.0f); + sxf = clamp((dx + 0.5f) * sc_w - 0.5f, 0.0f, (float)sw - 1.0f); + } + int y0 = (int)floor(syf), x0 = (int)floor(sxf); + int y1 = min(y0 + 1, (int)sh - 1), x1 = min(x0 + 1, (int)sw - 1); + float fy = syf - y0, fx = sxf - x0; + float v00 = (float)in[((size_t)y0*sw + x0)*E + e], v01 = (float)in[((size_t)y0*sw + x1)*E + e]; + float v10 = (float)in[((size_t)y1*sw + x0)*E + e], v11 = (float)in[((size_t)y1*sw + x1)*E + e]; + y[(size_t)pix*E + e] = (half)(v00*(1-fx)*(1-fy) + v01*fx*(1-fy) + v10*(1-fx)*fy + v11*fx*fy); +} + +kernel void conv1d_gen_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* bias [[buffer(2)]], device half* y [[buffer(3)]], + constant uint& Cin [[buffer(4)]], constant uint& L [[buffer(5)]], + constant uint& Cout [[buffer(6)]], constant uint& Lout [[buffer(7)]], + constant uint& K [[buffer(8)]], constant uint& stride [[buffer(9)]], + constant uint& has_bias [[buffer(10)]], constant uint& w_ck_co [[buffer(11)]], + uint3 gid [[thread_position_in_grid]]) { + uint lo = gid.x, co = gid.y, n = gid.z; + if (lo >= Lout || co >= Cout) return; + float acc = has_bias ? (float)bias[co] : 0.0f; + for (uint ci = 0; ci < Cin; ++ci) { + device const half* xr = x + ((size_t)n*Cin + ci)*L + (size_t)lo*stride; + for (uint k = 0; k < K; ++k) { + float wv = w_ck_co ? (float)w[((size_t)ci*K + k)*Cout + co] + : (float)w[((size_t)co*Cin + ci)*K + k]; + acc = fma((float)xr[k], wv, acc); + } + } + y[((size_t)n*Cout + co)*Lout + lo] = (half)acc; +} + +kernel void conv1d_nlc_dw_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* bias [[buffer(2)]], device half* y [[buffer(3)]], + constant uint& L [[buffer(4)]], constant uint& C [[buffer(5)]], + constant uint& K [[buffer(6)]], constant uint& dil [[buffer(7)]], + constant uint& pad [[buffer(8)]], constant uint& has_bias [[buffer(9)]], + uint3 gid [[thread_position_in_grid]]) { + uint c = gid.x, l = gid.y, n = gid.z; + if (c >= C || l >= L) return; + float acc = has_bias ? (float)bias[c] : 0.0f; + for (uint k = 0; k < K; ++k) { + long p = (long)l - (long)pad + (long)k * dil; + if (p >= 0 && p < (long)L) + acc = fma((float)x[((size_t)n*L + p)*C + c], (float)w[(size_t)c*K + k], acc); + } + y[((size_t)n*L + l)*C + c] = (half)acc; +} + +kernel void conv2d_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* bias [[buffer(2)]], device half* y [[buffer(3)]], + constant uint& Cin [[buffer(4)]], constant uint& H [[buffer(5)]], + constant uint& W [[buffer(6)]], constant uint& Cout [[buffer(7)]], + constant uint& Ho [[buffer(8)]], constant uint& Wo [[buffer(9)]], + constant uint& K [[buffer(10)]], constant uint& stride [[buffer(11)]], + constant uint& pad [[buffer(12)]], constant uint& dw [[buffer(13)]], + constant uint& has_bias [[buffer(14)]], + uint3 gid [[thread_position_in_grid]]) { + uint wo0 = gid.x * 4, ho = gid.y, nco = gid.z; + if (wo0 >= Wo || ho >= Ho) return; + uint n = nco / Cout, co = nco % Cout; + float acc[4]; + float bv = has_bias ? (float)bias[co] : 0.0f; + for (uint q = 0; q < 4; ++q) acc[q] = bv; + uint ci0 = dw ? co : 0, ci1 = dw ? co + 1 : Cin; + for (uint ci = ci0; ci < ci1; ++ci) { + for (uint kh = 0; kh < K; ++kh) { + long h = (long)ho * stride - (long)pad + kh; + if (h < 0 || h >= (long)H) continue; + device const half* xr = x + (((size_t)n*Cin + ci)*H + h)*W; + for (uint kw = 0; kw < K; ++kw) { + float wv = dw ? (float)w[((size_t)co*K + kh)*K + kw] + : (float)w[(((size_t)co*Cin + ci)*K + kh)*K + kw]; + for (uint q = 0; q < 4; ++q) { + long ww = (long)(wo0 + q) * stride - (long)pad + kw; + if (ww >= 0 && ww < (long)W && wo0 + q < Wo) + acc[q] = fma((float)xr[ww], wv, acc[q]); + } + } + } + } + for (uint q = 0; q < 4 && wo0 + q < Wo; ++q) + y[(((size_t)n*Cout + co)*Ho + ho)*Wo + wo0 + q] = (half)acc[q]; +} + +kernel void batchnorm_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* b [[buffer(2)]], device const half* rm [[buffer(3)]], + device const half* rv [[buffer(4)]], device half* y [[buffer(5)]], + constant uint& C [[buffer(6)]], constant uint& inner [[buffer(7)]], + constant float& eps [[buffer(8)]], constant uint& n [[buffer(9)]], + uint i [[thread_position_in_grid]]) { + if (i >= n) return; + uint c = (i / inner) % C; + float inv = 1.0f / precise::sqrt((float)rv[c] + eps); + y[i] = (half)(((float)x[i] - (float)rm[c]) * inv * (float)w[c] + (float)b[c]); +} + +kernel void groupnorm_f16(device const half* x [[buffer(0)]], device const half* w [[buffer(1)]], + device const half* b [[buffer(2)]], device half* y [[buffer(3)]], + constant uint& cpg [[buffer(4)]], constant uint& S [[buffer(5)]], + constant uint& C [[buffer(6)]], constant float& eps [[buffer(7)]], + uint2 tgp [[threadgroup_position_in_grid]], + uint2 tp [[thread_position_in_threadgroup]], uint2 ntp [[threads_per_threadgroup]], + threadgroup float* red [[threadgroup(0)]]) { + uint g = tgp.x, n = tgp.y, t = tp.x, nt = ntp.x; + size_t base = ((size_t)n*C + (size_t)g*cpg) * S; + uint count = cpg * S; + float sum = 0, sq = 0; + for (uint i = t; i < count; i += nt) { float v = (float)x[base + i]; sum += v; sq += v*v; } + red[t] = sum; threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint s2 = nt/2; s2 > 0; s2 >>= 1) { if (t < s2) red[t] += red[t+s2]; threadgroup_barrier(mem_flags::mem_threadgroup); } + sum = red[0]; threadgroup_barrier(mem_flags::mem_threadgroup); + red[t] = sq; threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint s2 = nt/2; s2 > 0; s2 >>= 1) { if (t < s2) red[t] += red[t+s2]; threadgroup_barrier(mem_flags::mem_threadgroup); } + sq = red[0]; + float mean = sum / count; + float inv = 1.0f / precise::sqrt(sq / count - mean*mean + eps); + for (uint i = t; i < count; i += nt) { + uint c = g*cpg + i / S; + y[base + i] = (half)(((float)x[base + i] - mean) * inv * (float)w[c] + (float)b[c]); + } +} + +kernel void bias_add_rows_f16(device half* y [[buffer(0)]], device const half* bias [[buffer(1)]], + constant uint& C [[buffer(2)]], constant uint& n [[buffer(3)]], + uint i [[thread_position_in_grid]]) { + if (i < n) y[i] = (half)((float)y[i] + (float)bias[i % C]); +} + +struct EwStep { int kind; int code; float p0; float p1; }; + +kernel void elemwise_chain_f16(device const uchar* in [[buffer(0)]], device uchar* y [[buffer(1)]], + constant EwStep* steps [[buffer(2)]], constant uint& nsteps [[buffer(3)]], + device const uchar* s0 [[buffer(4)]], device const uchar* s1 [[buffer(5)]], + device const uchar* s2 [[buffer(6)]], constant uint& n [[buffer(7)]], + constant uint& flags [[buffer(8)]], + constant uint& inner [[buffer(9)]], + uint gid [[thread_position_in_grid]]) { + uint i0 = gid * 4; + if (i0 >= n) return; + uint cnt = min(4u, n - i0); + float xv[4]; + bool in32 = (flags & 1u) != 0; + for (uint q = 0; q < cnt; ++q) + xv[q] = in32 ? ((device const float*)in)[i0+q] : (float)((device const half*)in)[i0+q]; + bool prec32 = in32; + for (uint s = 0; s < nsteps; ++s) { + EwStep st = steps[s]; + if (st.kind == 4) { prec32 = st.code != 0; continue; } + if (st.kind == 2) { + uint slot = (st.code >> 6) & 3u; + int op = st.code & 15; + bool rhs = (st.code & 16) != 0; + bool sf32 = (st.code & 32) != 0; + device const uchar* sp = slot == 0 ? s0 : slot == 1 ? s1 : s2; + uint bmode = (st.code >> 8) & 3u; + for (uint q = 0; q < cnt; ++q) { + uint si = i0 + q; + if (bmode == 1u) si = si / inner; + else if (bmode == 2u) si = si % inner; + float o = sf32 ? ((device const float*)sp)[si] : (float)((device const half*)sp)[si]; + float a = rhs ? o : xv[q], b = rhs ? xv[q] : o; + float r; + switch (op) { case 2: r=a-b; break; case 3: r=a*b; break; case 4: r=a/b; break; + case 5: r=(a!=b)?1.0f:0.0f; break; default: r=a+b; } + if (op == 1) r = clamp(r, -65500.0f, 65500.0f); + xv[q] = r; + } + } else if (st.kind == 0) { + for (uint q = 0; q < cnt; ++q) { + float x = xv[q]; + if (st.code == 0) x = gelu_tanh(x); + else if (st.code == 1) x = precise::tanh(x); + else if (st.code == 2) x = x/(1.0f+precise::exp(-x)); + else if (st.code == 4) x = 0.5f*x*(1.0f+erf_approx(x*0.70710678f)); + else if (st.code == 5) x = 1.0f/(1.0f+precise::exp(-x)); + else x = max(x, 0.0f); + xv[q] = x; + } + } else if (st.kind == 1) { + for (uint q = 0; q < cnt; ++q) xv[q] = scalar_apply(xv[q], st.p0, st.code); + } else { + for (uint q = 0; q < cnt; ++q) xv[q] = clamp(xv[q], st.p0, st.p1); + } + if (!prec32) for (uint q = 0; q < cnt; ++q) xv[q] = (float)(half)xv[q]; + } + if ((flags & 2u) != 0) { for (uint q = 0; q < cnt; ++q) ((device float*)y)[i0+q] = xv[q]; } + else { for (uint q = 0; q < cnt; ++q) ((device half*)y)[i0+q] = (half)xv[q]; } +} + +kernel void bcast_binary_rows_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], + device half* out [[buffer(2)]], constant uint* oshape [[buffer(3)]], + constant uint* astride [[buffer(4)]], constant uint* bstride [[buffer(5)]], + constant uint& ndim [[buffer(6)]], constant uint& rows [[buffer(7)]], constant int& op [[buffer(8)]], + constant uint& inner [[buffer(9)]], constant uint& ainner [[buffer(10)]], constant uint& binner [[buffer(11)]], + uint2 gid [[thread_position_in_grid]]) { + uint v = gid.x, row = gid.y; + if (v >= inner || row >= rows) return; + uint rem = row, ai = 0, bi = 0; + for (int d = int(ndim) - 1; d >= 0; --d) { uint c = rem % oshape[d]; rem /= oshape[d]; ai += c * astride[d]; bi += c * bstride[d]; } + float av = (float)a[ai + v * ainner], bv = (float)b[bi + v * binner], r; + switch(op){ case 2:r=av-bv;break; case 3:r=av*bv;break; case 4:r=av/bv;break; + case 5:r=(av!=bv)?1.0f:0.0f;break; default:r=av+bv; } + if (op==1) r=clamp(r,-65500.0f,65500.0f); + out[(size_t)row * inner + v] = (half)r; +} + +kernel void bcast_binary_f16(device const half* a [[buffer(0)]], device const half* b [[buffer(1)]], + device half* out [[buffer(2)]], constant uint* oshape [[buffer(3)]], + constant uint* astride [[buffer(4)]], constant uint* bstride [[buffer(5)]], + constant uint& ndim [[buffer(6)]], constant uint& total [[buffer(7)]], constant int& op [[buffer(8)]], + uint i [[thread_position_in_grid]]) { + if (i>=total) return; + uint rem=i, ai=0, bi=0; + for (int d=int(ndim)-1; d>=0; --d){ uint c=rem%oshape[d]; rem/=oshape[d]; ai+=c*astride[d]; bi+=c*bstride[d]; } + float av=(float)a[ai], bv=(float)b[bi], r; + switch(op){ case 2:r=av-bv;break; case 3:r=av*bv;break; case 4:r=av/bv;break; + case 5:r=(av!=bv)?1.0f:0.0f;break; default:r=av+bv; } + if (op==1) r=clamp(r,-65500.0f,65500.0f); + out[i]=(half)r; +} + +kernel void attn_decode_i8( + device const half* q [[buffer(0)]], + device const half* knew [[buffer(1)]], + device const half* vnew [[buffer(2)]], + device const char* kc [[buffer(3)]], + device const char* vc [[buffer(4)]], + device const float* ks [[buffer(5)]], + device const float* vs [[buffer(6)]], + device half* out [[buffer(7)]], + constant uint& num_q_heads [[buffer(8)]], constant uint& num_kv_heads [[buffer(9)]], + constant uint& head_dim [[buffer(10)]], constant uint& v_hdim [[buffer(11)]], + constant uint& history_len [[buffer(12)]], constant float& scale [[buffer(13)]], + constant uint& kv_start [[buffer(14)]], constant uint& kv_end [[buffer(15)]], + device float* part_o [[buffer(16)]], device float* part_ml [[buffer(17)]], + constant uint& nwg [[buffer(18)]], + uint tg [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]], threadgroup float* smem [[threadgroup(0)]]) +{ + uint h = tg / nwg, w = tg % nwg; + uint kvh = h / (num_q_heads / num_kv_heads); + uint ngK = (head_dim + 31u)/32u, ngV = (v_hdim + 31u)/32u; + uint nsg = T / 32u; + device const half* qh = q + (size_t)h*head_dim; + + float qreg[16]; + { uint i=0; for (uint d=lane; d 0.0f ? 1.0f/gl : 0.0f; + for (uint d=t; d0; s2>>=1){ if (t0; s2>>=1){ if (t0; s2>>=1){ if (t 0.0f ? 1.0f/gl : 0.0f; + for (uint d=t; d 0.0f ? 1.0f/gl : 0.0f; + for (uint d=t; d 0u) ? sinkN : 0u; + uint Wsr = S + ringR; + uint rstart, nactive; + if (ringR > 0u && kv_end > Wsr) { rstart = kv_end - ringR; nactive = Wsr; } + else { rstart = S; nactive = kv_end; } + device const half* qh = q + ((size_t)m*num_q_heads + h)*head_dim; + threadgroup float* red = sc + maxsc; + if (nactive == 0u) { + for (uint d=t; d 0u) { + float lmax = -INFINITY; + for (uint j = t; j < nactive; j += T) { + uint k = (j < S) ? j : rstart + (j - S); + float dot = 0; + if (k < history_len) { + uint slot = (ringR > 0u && k >= Wsr) ? (S + ((k - S) % ringR)) : k; + device const char* kk = kc + ((size_t)slot*num_kv_heads + kvh)*head_dim; + device const float* kss = ks + ((size_t)slot*num_kv_heads + kvh)*ngK; + for (uint d=0; d0;s>>=1){ if(t0;s>>=1){ if(t 0 ? 1.0f/red[0] : 0.0f; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint d = t; d < v_hdim; d += T) { + float acc=0; + for (uint j=0; j= Wsr) ? (S + ((k - S) % ringR)) : k; + device const char* vvv = vc + ((size_t)slot*num_kv_heads+kvh)*v_hdim; + device const float* vss = vs + ((size_t)slot*num_kv_heads+kvh)*ngV; + vv = (float)vvv[d]*vss[d/32]; + } else { + device const half* vvv = vnew + ((size_t)(k-history_len)*num_kv_heads+kvh)*v_hdim; + vv = (float)vvv[d]; + } + acc += sc[j]*vv; + } + out[((size_t)m*num_q_heads + h)*v_hdim + d] = (half)(acc*inv); + } + } else { + uint TILE = maxsc; + float rmax = -INFINITY, rsum = 0.0f; + float acc[8]; + for (uint a=0;a<8;++a) acc[a]=0.0f; + for (uint tile=0; tile0;s>>=1){ if(t0;s>>=1){ if(t 0 ? 1.0f/rsum : 0.0f; + uint ai=0; + for (uint d=t; d> 5, lane = tl & 31u; + uint h = sg >> 1, hf = sg & 1u, kvh = 0u; + uint vd0 = hf*256u; + uint m0 = tgx*QB; + uint total_keys = history_len + new_len; + + uint pos_last = q_pos0 + m0 + (QB-1u); + uint kv_end_tile = min(total_keys, pos_last + 1u); + simdgroup_matrix O[NFH]; + + for (uint f=0; f(0.f); + if (hf==0u && lane < QB) { mrun[h*QB+lane] = -INFINITY; lrun[h*QB+lane]=0.0f; } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint k0=0; k0 C0=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C1=C0,C2=C0,C3=C0,C4=C0,C5=C0,C6=C0,C7=C0; + for (uint c=0; c>5, d=i&31u; + Qs[h*QB*LD + r*LD + d] = (m0+r A,B0,B1,B2,B3,B4,B5,B6,B7; + simdgroup_load(A, &Qs[h*QB*LD + kk], LD); + simdgroup_load(B0,&Ks[kk*LDK + 0u], LDK); + simdgroup_load(B1,&Ks[kk*LDK + 8u], LDK); + simdgroup_load(B2,&Ks[kk*LDK + 16u],LDK); + simdgroup_load(B3,&Ks[kk*LDK + 24u],LDK); + simdgroup_load(B4,&Ks[kk*LDK + 32u],LDK); + simdgroup_load(B5,&Ks[kk*LDK + 40u],LDK); + simdgroup_load(B6,&Ks[kk*LDK + 48u],LDK); + simdgroup_load(B7,&Ks[kk*LDK + 56u],LDK); + simdgroup_multiply_accumulate(C0,A,B0,C0); + simdgroup_multiply_accumulate(C1,A,B1,C1); + simdgroup_multiply_accumulate(C2,A,B2,C2); + simdgroup_multiply_accumulate(C3,A,B3,C3); + simdgroup_multiply_accumulate(C4,A,B4,C4); + simdgroup_multiply_accumulate(C5,A,B5,C5); + simdgroup_multiply_accumulate(C6,A,B6,C6); + simdgroup_multiply_accumulate(C7,A,B7,C7); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (hf==0u) { + simdgroup_store(C0,&Ss[h*QB*BK + 0u], BK); + simdgroup_store(C1,&Ss[h*QB*BK + 8u], BK); + simdgroup_store(C2,&Ss[h*QB*BK + 16u],BK); + simdgroup_store(C3,&Ss[h*QB*BK + 24u],BK); + simdgroup_store(C4,&Ss[h*QB*BK + 32u],BK); + simdgroup_store(C5,&Ss[h*QB*BK + 40u],BK); + simdgroup_store(C6,&Ss[h*QB*BK + 48u],BK); + simdgroup_store(C7,&Ss[h*QB*BK + 56u],BK); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (hf==0u && lane < QB) { + uint r=lane, qpos=q_pos0+m0+r; + threadgroup float* srow=&Ss[h*QB*BK + r*BK]; + float tmax=-INFINITY; + for (uint k=0;k-INFINITY)?exp(srow[k]-mnew):0.0f; + srow[k]=e; tsum+=e; + } + lrun[h*QB+r]=lrun[h*QB+r]*resc+tsum; + mrun[h*QB+r]=mnew; resc8[h*QB+r]=resc; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (lane == 0u) { + int nr = 0; + for (uint r=0; r>3; Rsc[sg*QB*8u + e] *= resc8[h*QB+r]; } + simdgroup_barrier(mem_flags::mem_threadgroup); + simdgroup_load(O[f], &Rsc[sg*QB*8u], 8u); + simdgroup_barrier(mem_flags::mem_threadgroup); + } + } + + if (hf==0u) for (uint i=lane; i>5, d=i&31u; + half val=(half)0; + if (k P,V0; + simdgroup_load(P, &Ps[h*QB*LDP + kk], LDP); + simdgroup_load(V0, &Vs[hf*BK*LD + kk*LD + fc], LD); + simdgroup_multiply_accumulate(O[f], P, V0, O[f]); + } + } + } + } + for (uint f=0; f>3, d=e&7u; + if (m0+r0?1.0f/lrun[h*QB+r]:0.0f; + out[((size_t)(m0+r)*num_q_heads + h)*VD + (vd0 + f*8u + d)] = (half)(Rsc[sg*QB*8u + e]*inv); + } + } + simdgroup_barrier(mem_flags::mem_threadgroup); + } +} + +kernel void attn_prefill_mma_hd256( + device const half* q [[buffer(0)]], + device const half* knew [[buffer(1)]], + device const half* vnew [[buffer(2)]], + device const char* kc [[buffer(3)]], + device const char* vc [[buffer(4)]], + device const float* ks [[buffer(5)]], + device const float* vs [[buffer(6)]], + device half* out [[buffer(7)]], + constant uint& num_q_heads [[buffer(8)]], constant uint& num_kv_heads [[buffer(9)]], + constant uint& head_dim [[buffer(10)]], constant uint& v_hdim [[buffer(11)]], + constant uint& history_len [[buffer(12)]], constant float& scale [[buffer(13)]], + constant uint& q_pos0 [[buffer(14)]], constant uint& new_len [[buffer(15)]], + constant uint& Mtot [[buffer(16)]], + constant uint& sinkN [[buffer(17)]], constant uint& ringR [[buffer(18)]], + uint tgx [[threadgroup_position_in_grid]], uint tl [[thread_index_in_threadgroup]]) +{ + const uint NSG=8u, LD=40u, BK=64u, QB=8u, HD=256u, VD=256u, NFH=32u; + const uint LDK=72u, LDP=72u; + threadgroup float poolA[4096]; + threadgroup half poolB[4608]; + threadgroup float Rsc[NSG*QB*8u]; + threadgroup float mrun[8u*QB]; + threadgroup float lrun[8u*QB]; + threadgroup float resc8[8u*QB]; + threadgroup int need_rescale[NSG]; + + threadgroup half* Ks = (threadgroup half*)poolA; + threadgroup float* Ss = poolA; + threadgroup half* Vs = (threadgroup half*)poolA; + threadgroup half* Qs = poolB; + threadgroup half* Ps = poolB; + + uint sg = tl >> 5, lane = tl & 31u; + uint h = sg, kvh = 0u; + uint m0 = tgx*QB; + uint total_keys = history_len + new_len; + + uint S = (ringR > 0u) ? sinkN : 0u; + uint R = ringR, Wsr = S + R; + uint pos_first = q_pos0 + m0; + uint pos_last = q_pos0 + m0 + (QB-1u); + uint kv_end_tile = min(total_keys, pos_last + 1u); + bool windowed = (R > 0u && pos_first >= Wsr); + uint rstart = windowed ? (pos_first + 1u - R) : 0u; + uint nactive = windowed ? (S + (kv_end_tile - rstart)) : kv_end_tile; + simdgroup_matrix O[NFH]; + + for (uint f=0; f(0.f); + if (lane < QB) { mrun[h*QB+lane] = -INFINITY; lrun[h*QB+lane]=0.0f; } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint k0=0; k0 C0=make_filled_simdgroup_matrix(0.f); + simdgroup_matrix C1=C0,C2=C0,C3=C0,C4=C0,C5=C0,C6=C0,C7=C0; + for (uint c=0; c0u && kk>=Wsr)?(S+(kk-S)%R):kk; + device const char* kp=kc+((size_t)slot*num_kv_heads+kvh)*HD; + device const float* sp=ks+((size_t)slot*num_kv_heads+kvh)*(HD/32u); + val=(half)((float)kp[c+d]*sp[(c+d)/32u]); + } else { + device const half* kp=knew+((size_t)(kk-history_len)*num_kv_heads+kvh)*HD; + val=kp[c+d]; + } + } + Ks[d*LDK + k]=val; + } + for (uint i=lane; i>5, d=i&31u; + Qs[h*QB*LD + r*LD + d] = (m0+r A,B0,B1,B2,B3,B4,B5,B6,B7; + simdgroup_load(A, &Qs[h*QB*LD + kk], LD); + simdgroup_load(B0,&Ks[kk*LDK + 0u], LDK); + simdgroup_load(B1,&Ks[kk*LDK + 8u], LDK); + simdgroup_load(B2,&Ks[kk*LDK + 16u],LDK); + simdgroup_load(B3,&Ks[kk*LDK + 24u],LDK); + simdgroup_load(B4,&Ks[kk*LDK + 32u],LDK); + simdgroup_load(B5,&Ks[kk*LDK + 40u],LDK); + simdgroup_load(B6,&Ks[kk*LDK + 48u],LDK); + simdgroup_load(B7,&Ks[kk*LDK + 56u],LDK); + simdgroup_multiply_accumulate(C0,A,B0,C0); + simdgroup_multiply_accumulate(C1,A,B1,C1); + simdgroup_multiply_accumulate(C2,A,B2,C2); + simdgroup_multiply_accumulate(C3,A,B3,C3); + simdgroup_multiply_accumulate(C4,A,B4,C4); + simdgroup_multiply_accumulate(C5,A,B5,C5); + simdgroup_multiply_accumulate(C6,A,B6,C6); + simdgroup_multiply_accumulate(C7,A,B7,C7); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + simdgroup_store(C0,&Ss[h*QB*BK + 0u], BK); + simdgroup_store(C1,&Ss[h*QB*BK + 8u], BK); + simdgroup_store(C2,&Ss[h*QB*BK + 16u],BK); + simdgroup_store(C3,&Ss[h*QB*BK + 24u],BK); + simdgroup_store(C4,&Ss[h*QB*BK + 32u],BK); + simdgroup_store(C5,&Ss[h*QB*BK + 40u],BK); + simdgroup_store(C6,&Ss[h*QB*BK + 48u],BK); + simdgroup_store(C7,&Ss[h*QB*BK + 56u],BK); + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (lane < QB) { + uint r=lane, qpos=q_pos0+m0+r; + threadgroup float* srow=&Ss[h*QB*BK + r*BK]; + float tmax=-INFINITY; + for (uint k=0;kqpos); + float s = att ? srow[k]*scale : -INFINITY; + srow[k]=s; tmax=max(tmax,s); + } + float mo=mrun[h*QB+r], mnew=max(mo,tmax); + float resc=exp(mo-mnew), tsum=0.0f; + for (uint k=0;k-INFINITY)?exp(srow[k]-mnew):0.0f; + srow[k]=e; tsum+=e; + } + lrun[h*QB+r]=lrun[h*QB+r]*resc+tsum; + mrun[h*QB+r]=mnew; resc8[h*QB+r]=resc; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (lane == 0u) { + int nr = 0; + for (uint r=0; r>3; Rsc[sg*QB*8u + e] *= resc8[h*QB+r]; } + simdgroup_barrier(mem_flags::mem_threadgroup); + simdgroup_load(O[f], &Rsc[sg*QB*8u], 8u); + simdgroup_barrier(mem_flags::mem_threadgroup); + } + } + + for (uint i=lane; i>5, d=i&31u; + half val=(half)0; + if (k0u && kk>=Wsr)?(S+(kk-S)%R):kk; + device const char* vp=vc+((size_t)slot*num_kv_heads+kvh)*VD; + device const float* sp=vs+((size_t)slot*num_kv_heads+kvh)*(VD/32u); + val=(half)((float)vp[c+d]*sp[(c+d)/32u]); + } else { + device const half* vp=vnew+((size_t)(kk-history_len)*num_kv_heads+kvh)*VD; + val=vp[c+d]; + } + } + Vs[k*LD + d]=val; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint fc=0; fc<32u; fc+=8u){ + uint f = jc/8u + fc/8u; + for (uint kk=0; kk P,V0; + simdgroup_load(P, &Ps[h*QB*LDP + kk], LDP); + simdgroup_load(V0, &Vs[kk*LD + fc], LD); + simdgroup_multiply_accumulate(O[f], P, V0, O[f]); + } + } + } + } + for (uint f=0; f>3, d=e&7u; + if (m0+r0?1.0f/lrun[h*QB+r]:0.0f; + out[((size_t)(m0+r)*num_q_heads + h)*VD + (f*8u + d)] = (half)(Rsc[sg*QB*8u + e]*inv); + } + } + simdgroup_barrier(mem_flags::mem_threadgroup); + } +} + +kernel void kv_append_i8_m(device const half* src [[buffer(0)]], device char* int8base [[buffer(1)]], + device float* scalebase [[buffer(2)]], constant uint& kv_heads [[buffer(3)]], + constant uint& hdim [[buffer(4)]], constant uint& current_len [[buffer(5)]], + constant uint& group_size [[buffer(6)]], constant uint& M [[buffer(7)]], + uint gid [[thread_position_in_grid]]) { + uint num_groups = (hdim + group_size - 1)/group_size, per = kv_heads*num_groups; + if (gid >= M*per) return; + uint i = gid / per, hg = gid % per, h = hg / num_groups, g = hg % num_groups; + uint gstart = g*group_size, gcount = min(group_size, hdim - gstart); + device const half* hs = src + (size_t)i*kv_heads*hdim + (size_t)h*hdim + gstart; + float maxabs = 0; + for (uint k=0;k= N) return; + float acc = 0; + if (tr != 0u) { + device const half* wr = w + (size_t)n * K; + for (uint k = lane; k < K; k += 32u) acc = fma((float)x[k], (float)wr[k], acc); + } else { + for (uint k = lane; k < K; k += 32u) acc = fma((float)x[k], (float)w[(size_t)k * N + n], acc); + } + acc = simd_sum(acc); + if (lane == 0) y[n] = (half)((float)(half)acc + (float)bias[n]); +} +kernel void rel_pos_bias_f16(device const half* q [[buffer(0)]], device const half* r [[buffer(1)]], + device half* y [[buffer(2)]], constant uint& T [[buffer(3)]], constant uint& H [[buffer(4)]], + constant uint& D [[buffer(5)]], constant uint& rbs [[buffer(6)]], + constant float& scale [[buffer(7)]], + uint3 gid [[thread_position_in_grid]]) { + uint tj = gid.x, h = gid.y, b = gid.z; + if (tj >= T * T || h >= H) return; + uint t = tj / T, j = tj % T; + device const half* qv = q + ((size_t)b * T + t) * H * D + h * D; + uint rel = (T - 1u) - t + j; + device const half* rv = r + (size_t)b * rbs + (size_t)rel * H * D + h * D; + float acc = 0; + for (uint d = 0; d < D; ++d) acc = fma((float)qv[d], (float)rv[d], acc); + y[((size_t)b * H + h) * T * T + (size_t)t * T + j] = (half)(acc * scale); +} +kernel void conv_cache_append_f16(device const uchar* src [[buffer(0)]], + device half* ring [[buffer(1)]], device half* out [[buffer(2)]], + constant uint& hd [[buffer(3)]], constant uint& ws [[buffer(4)]], + constant uint& nnew [[buffer(5)]], constant uint& head0 [[buffer(6)]], + constant uint& count_new [[buffer(7)]], constant uint& num_rows [[buffer(8)]], + constant uint& src_f32 [[buffer(9)]], + uint2 gid [[thread_position_in_grid]]) { + uint x = gid.x, y = gid.y; + if (x >= hd || y >= ws + nnew) return; + uint start_row = num_rows - nnew; + if (y >= ws) { + uint i = y - ws; + uint sidx = (start_row + i) * hd + x; + half v = src_f32 ? (half)((device const float*)src)[sidx] : ((device const half*)src)[sidx]; + ring[((head0 + i) % ws) * hd + x] = v; + return; + } + uint pad = ws - count_new; + if (y < pad) { out[y * hd + x] = 0.0h; return; } + uint a = count_new - 1u - (y - pad); + half v; + if (a < nnew) { + uint sidx = (num_rows - 1u - a) * hd + x; + v = src_f32 ? (half)((device const float*)src)[sidx] : ((device const half*)src)[sidx]; + } else { + uint b = a - nnew; + v = ring[((head0 + 2u * ws - 1u - b) % ws) * hd + x]; + } + out[y * hd + x] = v; +} +kernel void kv_append_ring_i8_m(device const half* src [[buffer(0)]], device char* int8base [[buffer(1)]], + device float* scalebase [[buffer(2)]], constant uint& kv_heads [[buffer(3)]], + constant uint& hdim [[buffer(4)]], constant uint& current_len [[buffer(5)]], + constant uint& group_size [[buffer(6)]], constant uint& M [[buffer(7)]], + constant uint& sink [[buffer(8)]], constant uint& W [[buffer(9)]], + uint gid [[thread_position_in_grid]]) { + uint num_groups = (hdim + group_size - 1)/group_size, per = kv_heads*num_groups; + if (gid >= M*per) return; + uint i = gid / per, hg = gid % per, h = hg / num_groups, g = hg % num_groups; + uint gstart = g*group_size, gcount = min(group_size, hdim - gstart); + device const half* hs = src + (size_t)i*kv_heads*hdim + (size_t)h*hdim + gstart; + float maxabs = 0; + for (uint k=0;k sink) ? (W - sink) : 1u; + uint slot = (pos < W) ? pos : sink + ((pos - sink) % R); + if (pos >= sink && pos + R < current_len + M) return; + device char* dst = int8base + (size_t)slot*i8s + (size_t)h*hdim + gstart; + for (uint k=0;k= U.vocab) continue; + float v = (float)logits[id]; + logits[id] = (half)(v > 0.0f ? v/U.penalty : v*U.penalty); + } + } + threadgroup_barrier(mem_flags::mem_device); + if (t == 0u && U.suppress_flag != 0u && U.suppress_id < U.vocab) logits[U.suppress_id] = half(-65504.0f); +} + +kernel void softcap_f16(device const half* in [[buffer(0)]], device half* y [[buffer(1)]], + constant uint& n [[buffer(2)]], constant float& cap [[buffer(3)]], + uint i [[thread_position_in_grid]]) { + if (i>=n) return; + half v1 = (half)((float)in[i]/cap); + half v2 = (half)precise::tanh((float)v1); + y[i]=(half)((float)v2*cap); +} + +kernel void argmax_part(device const half* logits [[buffer(0)]], + device float* part [[buffer(1)]], + constant uint& V [[buffer(2)]], + constant uint& chunk [[buffer(3)]], + device const float* bias [[buffer(4)]], + constant uint& has_bias [[buffer(5)]], + uint p [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* sb [[threadgroup(0)]], + threadgroup uint* si [[threadgroup(1)]], + threadgroup float* ss [[threadgroup(2)]]) { + uint lo = p*chunk, hi = min(V, lo+chunk); + float b = -INFINITY, s = -INFINITY; uint bi = lo; + for (uint i = lo+t; i < hi; i += T) { + float v = (float)logits[i]; + if (has_bias != 0u) v += bias[i]; + if (v > b) { s = b; b = v; bi = i; } + else if (v > s) { s = v; } + } + sb[t] = b; si[t] = bi; ss[t] = s; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = T >> 1; stride > 0u; stride >>= 1) { + if (t < stride) { + float ab = sb[t], asx = ss[t]; uint ai = si[t]; + float bb = sb[t+stride], bsx = ss[t+stride]; uint bidx = si[t+stride]; + if (ab > bb || (ab == bb && ai <= bidx)) { sb[t] = ab; si[t] = ai; ss[t] = max(asx, bb); } + else { sb[t] = bb; si[t] = bidx; ss[t] = max(bsx, ab); } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (t == 0u) { part[p*3u] = sb[0]; part[p*3u+1u] = ss[0]; part[p*3u+2u] = (float)si[0]; } +} + +kernel void argmax_final(device const float* part [[buffer(0)]], + device float* out3 [[buffer(1)]], + constant uint& NP [[buffer(2)]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* sb [[threadgroup(0)]], + threadgroup uint* si [[threadgroup(1)]], + threadgroup float* ss [[threadgroup(2)]]) { + float b = -INFINITY, s = -INFINITY; uint bi = 0u; + for (uint i = t; i < NP; i += T) { + float pb = part[i*3u], ps = part[i*3u+1u]; uint pi = (uint)part[i*3u+2u]; + if (pb > b || (pb == b && pi <= bi)) { s = max(s, b); b = pb; bi = pi; s = max(s, ps); } + else { s = max(s, pb); } + } + sb[t] = b; si[t] = bi; ss[t] = s; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = T >> 1; stride > 0u; stride >>= 1) { + if (t < stride) { + float ab = sb[t], asx = ss[t]; uint ai = si[t]; + float bb = sb[t+stride], bsx = ss[t+stride]; uint bidx = si[t+stride]; + if (ab > bb || (ab == bb && ai <= bidx)) { sb[t] = ab; si[t] = ai; ss[t] = max(asx, bb); } + else { sb[t] = bb; si[t] = bidx; ss[t] = max(bsx, ab); } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (t == 0u) { out3[0] = sb[0]; out3[1] = ss[0]; out3[2] = (float)si[0]; } +} + +kernel void argmax_logits(device const half* logits [[buffer(0)]], + device float* out3 [[buffer(1)]], + constant uint& V [[buffer(2)]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* sb [[threadgroup(0)]], + threadgroup uint* si [[threadgroup(1)]], + threadgroup float* ss [[threadgroup(2)]]) { + float b = -INFINITY, s = -INFINITY; uint bi = 0u; + for (uint i = t; i < V; i += T) { + float v = (float)logits[i]; + if (v > b) { s = b; b = v; bi = i; } + else if (v > s) { s = v; } + } + sb[t] = b; si[t] = bi; ss[t] = s; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = T >> 1; stride > 0u; stride >>= 1) { + if (t < stride) { + float ab = sb[t], asx = ss[t]; uint ai = si[t]; + float bb = sb[t+stride], bsx = ss[t+stride]; uint bidx = si[t+stride]; + if (ab > bb || (ab == bb && ai <= bidx)) { sb[t] = ab; si[t] = ai; ss[t] = max(asx, bb); } + else { sb[t] = bb; si[t] = bidx; ss[t] = max(bsx, ab); } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (t == 0u) { out3[0] = sb[0]; out3[1] = ss[0]; out3[2] = (float)si[0]; } +} + +kernel void topk_rows_f16(device const half* in [[buffer(0)]], + device float* out [[buffer(1)]], + constant uint& F [[buffer(2)]], + constant uint& k [[buffer(3)]], + constant uint& B [[buffer(4)]], + uint row [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]]) { + device const half* x = in + (size_t)row*F; + device float* idx_out = out + (size_t)row*k; + device float* val_out = out + (size_t)B*k + (size_t)row*k; + for (uint j = 0u; j < k; ++j) { + float best = -INFINITY; uint bi = 0xFFFFFFFFu; + for (uint i = lane; i < F; i += 32u) { + bool taken = false; + for (uint p = 0u; p < j; ++p) if ((uint)idx_out[p] == i) taken = true; + float v = (float)x[i]; + if (!taken && (v > best || (v == best && i < bi))) { best = v; bi = i; } + } + float gb = simd_max(best); + uint cand = (best == gb) ? bi : 0xFFFFFFFFu; + uint gi = simd_min(cand); + if (lane == 0u) { idx_out[j] = (float)gi; val_out[j] = gb; } + simdgroup_barrier(mem_flags::mem_device); + } +} + +kernel void cq4_moe_transform( + device const half* x [[buffer(0)]], + device const float* topk [[buffer(1)]], + device const half* recip_c [[buffer(2)]], + device const char* lsign_c [[buffer(3)]], + device const char* rsign_c [[buffer(4)]], + device const uint* perm_c [[buffer(5)]], + device half* code [[buffer(6)]], + constant uint& K [[buffer(7)]], + constant uint& k_valid [[buffer(8)]], + constant uint& xz_stride [[buffer(9)]], + constant uint& xy_stride [[buffer(10)]], + constant uint& topk_k [[buffer(11)]], + constant uint& estride [[buffer(12)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + threadgroup float* zmem [[threadgroup(0)]]) +{ + const uint g = tgp.x, slot = tgp.y, tok = tgp.z; + const uint e = (uint)(topk[(size_t)tok*topk_k + slot] + 0.5f); + device const half* recip = (device const half*)((device const uchar*)recip_c + (size_t)e*estride); + device const char* lsign = lsign_c + (size_t)e*estride; + device const char* rsign = rsign_c + (size_t)e*estride; + device const uint* perm = (device const uint*)((device const uchar*)perm_c + (size_t)e*estride); + device const half* xs = x + (size_t)tok*xz_stride + (size_t)slot*xy_stride; + uint b = g*128u + lane*4u; + uint k = lane*4u; + float xv[4]; + #pragma clang loop unroll(full) + for (uint j = 0u; j < 4u; ++j) { + float v = (b+j < k_valid) ? (float)xs[b+j] : 0.0f; + xv[j] = v*(float)recip[b+j]*(float)lsign[k+j]; + } + float x0 = xv[0], x1 = xv[1], x2 = xv[2], x3 = xv[3]; + cq4_hada128(x0, x1, x2, x3, lane); + zmem[k+0] = x0*(float)rsign[k+0]; zmem[k+1] = x1*(float)rsign[k+1]; + zmem[k+2] = x2*(float)rsign[k+2]; zmem[k+3] = x3*(float)rsign[k+3]; + threadgroup_barrier(mem_flags::mem_threadgroup); + device half* co = code + ((size_t)tok*topk_k + slot)*K + g*128u; + co[k+0] = (half)zmem[perm[k+0]]; co[k+1] = (half)zmem[perm[k+1]]; + co[k+2] = (half)zmem[perm[k+2]]; co[k+3] = (half)zmem[perm[k+3]]; +} + +kernel void cq4_moe_gemv_up( + device const half* code1 [[buffer(0)]], + device const half* code3 [[buffer(1)]], + device const float* topk [[buffer(2)]], + device const uchar* pk1 [[buffer(3)]], + device const half* nm1 [[buffer(4)]], + device const half* cb1 [[buffer(5)]], + device const uchar* pk3 [[buffer(6)]], + device const half* nm3 [[buffer(7)]], + device const half* cb3 [[buffer(8)]], + device half* y [[buffer(9)]], + constant uint& K [[buffer(10)]], + constant uint& N [[buffer(11)]], + constant uint& act [[buffer(12)]], + constant uint& topk_k [[buffer(13)]], + constant uint& estride1 [[buffer(14)]], + constant uint& estride3 [[buffer(15)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]]) +{ + const uint slot = tgp.y, tok = tgp.z; + const uint slotg = tok*topk_k + slot; + const uint e = (uint)(topk[(size_t)tok*topk_k + slot] + 0.5f); + const uint ng = K/128u; + threadgroup float c1[16], c3[16]; + if (tl < 16u) { + c1[tl] = (float)((device const half*)((device const uchar*)cb1 + (size_t)e*estride1))[tl]; + c3[tl] = (float)((device const half*)((device const uchar*)cb3 + (size_t)e*estride3))[tl]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + uint n0 = (tgp.x*8u + sgid)*4u; + if (n0 >= N) return; + float gv[4] = {0,0,0,0}, uv[4] = {0,0,0,0}; + for (int m = 0; m < 2; ++m) { + device const uchar* pb = (m==0) ? pk1 + (size_t)e*estride1 : pk3 + (size_t)e*estride3; + device const half* nb = (m==0) ? (device const half*)((device const uchar*)nm1 + (size_t)e*estride1) + : (device const half*)((device const uchar*)nm3 + (size_t)e*estride3); + device const half* cd = (m==0) ? code1 + (size_t)slotg*K : code3 + (size_t)slotg*K; + threadgroup float* cb = (m==0) ? c1 : c3; + float a0=0, a1=0, a2=0, a3=0; + for (uint base = lane*16u; base < K; base += 512u) { + uint gg = base/128u, off = base - gg*128u; + device const half4* cp = (device const half4*)(cd + gg*128u + off); + half4 c0=cp[0], c1v=cp[1], c2v=cp[2], c3v=cp[3]; + device const uchar* pan = pb + ((size_t)(n0>>2)*ng+gg)*256u + (off>>4)*32u; + uint2 A01 = ((device const uint2*)pan)[0]; + uint2 A23 = ((device const uint2*)(pan+8u))[0]; + uint2 B01 = ((device const uint2*)(pan+16u))[0]; + uint2 B23 = ((device const uint2*)(pan+24u))[0]; + size_t nb4 = ((size_t)(n0>>2)*ng+gg)<<2; + a0 += (float)nb[nb4+0u]*cq4_dot64_ilw(c0, c1v, c2v, c3v, A01.x, B01.x, cb); + a1 += (float)nb[nb4+1u]*cq4_dot64_ilw(c0, c1v, c2v, c3v, A01.y, B01.y, cb); + a2 += (float)nb[nb4+2u]*cq4_dot64_ilw(c0, c1v, c2v, c3v, A23.x, B23.x, cb); + a3 += (float)nb[nb4+3u]*cq4_dot64_ilw(c0, c1v, c2v, c3v, A23.y, B23.y, cb); + } + a0 = simd_sum(a0); a1 = simd_sum(a1); a2 = simd_sum(a2); a3 = simd_sum(a3); + if (m == 0) { gv[0]=a0; gv[1]=a1; gv[2]=a2; gv[3]=a3; } + else { uv[0]=a0; uv[1]=a1; uv[2]=a2; uv[3]=a3; } + } + if (lane == 0u) { + device half* ys = y + (size_t)slotg*N; + for (uint r = 0u; r < 4u && n0+r < N; ++r) { + float gh = (float)(half)gv[r]; + float av = (act == 1u) ? gelu_tanh(gh) : gh/(1.0f + precise::exp(-gh)); + ys[n0+r] = (half)((float)(half)av*(float)(half)uv[r]); + } + } +} + +kernel void cq4_moe_transform2( + device const half* x [[buffer(0)]], + device const float* topk [[buffer(1)]], + device const half* recip_c1 [[buffer(2)]], + device const char* lsign_c1 [[buffer(3)]], + device const char* rsign_c1 [[buffer(4)]], + device const uint* perm_c1 [[buffer(5)]], + device const half* recip_c3 [[buffer(6)]], + device const char* lsign_c3 [[buffer(7)]], + device const char* rsign_c3 [[buffer(8)]], + device const uint* perm_c3 [[buffer(9)]], + device half* code1 [[buffer(10)]], + device half* code3 [[buffer(11)]], + constant uint& K [[buffer(12)]], + constant uint& topk_k [[buffer(13)]], + constant uint& estride1 [[buffer(14)]], + constant uint& estride3 [[buffer(15)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]], + threadgroup float* zmem [[threadgroup(0)]]) +{ + const uint g = tgp.x, tok = tgp.z; + const bool w3 = tgp.y >= topk_k; + const uint slot = w3 ? tgp.y - topk_k : tgp.y; + const uint e = (uint)(topk[(size_t)tok*topk_k + slot] + 0.5f); + const uint es = w3 ? estride3 : estride1; + device const half* recip = (device const half*)((device const uchar*)(w3 ? recip_c3 : recip_c1) + (size_t)e*es); + device const char* lsign = (w3 ? lsign_c3 : lsign_c1) + (size_t)e*es; + device const char* rsign = (w3 ? rsign_c3 : rsign_c1) + (size_t)e*es; + device const uint* perm = (device const uint*)((device const uchar*)(w3 ? perm_c3 : perm_c1) + (size_t)e*es); + device const half* xs = x + (size_t)tok*K; + uint b = g*128u + lane*4u; + uint k = lane*4u; + float x0=(float)xs[b+0]*(float)recip[b+0]*(float)lsign[k+0]; + float x1=(float)xs[b+1]*(float)recip[b+1]*(float)lsign[k+1]; + float x2=(float)xs[b+2]*(float)recip[b+2]*(float)lsign[k+2]; + float x3=(float)xs[b+3]*(float)recip[b+3]*(float)lsign[k+3]; + cq4_hada128(x0, x1, x2, x3, lane); + zmem[k+0]=x0*(float)rsign[k+0]; zmem[k+1]=x1*(float)rsign[k+1]; + zmem[k+2]=x2*(float)rsign[k+2]; zmem[k+3]=x3*(float)rsign[k+3]; + threadgroup_barrier(mem_flags::mem_threadgroup); + device half* co = (w3 ? code3 : code1) + ((size_t)tok*topk_k + slot)*K + g*128u; + co[k+0]=(half)zmem[perm[k+0]]; co[k+1]=(half)zmem[perm[k+1]]; + co[k+2]=(half)zmem[perm[k+2]]; co[k+3]=(half)zmem[perm[k+3]]; +} + +kernel void cq4_moe_gemv_down_acc( + device const half* code2 [[buffer(0)]], + device const float* topk [[buffer(1)]], + device const half* probs [[buffer(2)]], + device const uchar* pk2 [[buffer(3)]], + device const half* nm2 [[buffer(4)]], + device const half* cb2 [[buffer(5)]], + device half* out [[buffer(6)]], + constant uint& K [[buffer(7)]], + constant uint& N [[buffer(8)]], + constant uint& topk_k [[buffer(9)]], + constant uint& normalize [[buffer(10)]], + constant float& eps [[buffer(11)]], + constant float& scaling [[buffer(12)]], + constant uint& E [[buffer(13)]], + constant uint& estride [[buffer(14)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint sgid [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]], + uint tl [[thread_index_in_threadgroup]], + threadgroup float* cbs [[threadgroup(0)]]) +{ + const uint tok = tgp.z; + device const float* tk = topk + (size_t)tok*topk_k; + device const half* pr = probs + (size_t)tok*E; + uint order[16]; + for (uint j = 0u; j < topk_k; ++j) order[j] = j; + for (uint a = 1u; a < topk_k; ++a) { + uint v = order[a]; + uint ev = (uint)(tk[v] + 0.5f); + uint b = a; + while (b > 0u && (uint)(tk[order[b-1u]] + 0.5f) > ev) { order[b] = order[b-1u]; --b; } + order[b] = v; + } + if (tl < topk_k*16u) { + uint s = tl >> 4, ci = tl & 15u; + uint e = (uint)(tk[s] + 0.5f); + cbs[tl] = (float)((device const half*)((device const uchar*)cb2 + (size_t)e*estride))[ci]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + float denom = 1.0f; + if (normalize != 0u) { + float s = 0.0f; + for (uint j = 0u; j < topk_k; ++j) s += (float)pr[(uint)(tk[j] + 0.5f)]; + denom = s + eps; + } + const uint ng = K/128u; + uint n0 = (tgp.x*8u + sgid)*2u; + if (n0 >= N) return; + half res[2] = {0.0h, 0.0h}; + for (uint j = 0u; j < topk_k; ++j) { + uint slot = order[j]; + uint e = (uint)(tk[slot] + 0.5f); + float p = (float)pr[e]; + if (p <= 0.0f) continue; + float w = p/denom*scaling; + device const uchar* pbase = pk2 + (size_t)e*estride; + device const half* nbase = (device const half*)((device const uchar*)nm2 + (size_t)e*estride); + device const half* cd = code2 + ((size_t)tok*topk_k + slot)*K; + threadgroup const float* cb = cbs + (size_t)slot*16u; + float acc[2] = {0, 0}; + for (uint base = lane*16u; base < K; base += 512u) { + uint gg = base/128u, off = base - gg*128u; + device const half4* cp = (device const half4*)(cd + gg*128u + off); + half4 c0=cp[0], c1=cp[1], c2=cp[2], c3=cp[3]; + device const uchar* pan = pbase + ((size_t)(n0>>2)*ng+gg)*256u + (n0&3u)*4u + (off>>4)*32u; + uint2 Aw = ((device const uint2*)pan)[0]; + uint2 Bw = ((device const uint2*)(pan+16u))[0]; + #pragma clang loop unroll(full) + for (uint r = 0u; r < 2u; ++r) { + float pv = cq4_dot64_ilw(c0, c1, c2, c3, r?Aw.y:Aw.x, r?Bw.y:Bw.x, cb); + acc[r] += (float)nbase[(((size_t)(n0>>2)*ng+gg)<<2)+(n0&3u)+r]*pv; + } + } + #pragma clang loop unroll(full) + for (uint r = 0u; r < 2u; ++r) { + float a = simd_sum(acc[r]); + res[r] = (half)((float)res[r] + (float)(half)a*w); + } + } + if (lane == 0u) { + device half* ys = out + (size_t)tok*N; + for (uint r = 0u; r < 2u && n0+r < N; ++r) ys[n0+r] = res[r]; + } +} + +kernel void rope_pair_f16( + device const half* x [[buffer(0)]], + device const half* c [[buffer(1)]], + device const half* s [[buffer(2)]], + device half* y [[buffer(3)]], + constant uint& H [[buffer(4)]], + constant uint& D [[buffer(5)]], + uint2 g [[thread_position_in_grid]]) +{ + uint d = g.x, h = g.y; + if (d >= D || h >= H) return; + size_t idx = (size_t)h*D + d; + half xv = x[idx]; + half rh = (d < D/2u) ? (half)(-x[idx + D/2u]) : x[idx - D/2u]; + y[idx] = xv*c[d] + rh*s[d]; +} + +kernel void rms_norm_scale_f16( + device const half* in [[buffer(0)]], + device const half* w [[buffer(1)]], + device half* y [[buffer(2)]], + constant uint& D [[buffer(3)]], + constant float& eps [[buffer(4)]], + constant float& oscale [[buffer(5)]], + uint row [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* red [[threadgroup(0)]]) +{ + device const half* x = in + (size_t)row*D; + device half* o = y + (size_t)row*D; + float ss = 0; + for (uint i = t; i < D; i += T) { float v = (float)x[i]; ss += v*v; } + red[t] = ss; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint s = T/2; s > 0; s >>= 1) { + if (t < s) red[t] += red[t+s]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + float inv = rsqrt(red[0]/(float)D + eps); + for (uint i = t; i < D; i += T) { + half y0 = (half)((float)x[i]*inv*(float)w[i]); + o[i] = (half)((float)y0*oscale); + } +} + +kernel void softmax_topk_f16( + device const half* in [[buffer(0)]], + device half* probs [[buffer(1)]], + device float* tk [[buffer(2)]], + constant uint& E [[buffer(3)]], + constant uint& k [[buffer(4)]], + constant uint& B [[buffer(5)]], + constant float& scale [[buffer(6)]], + uint row [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]]) +{ + device const half* x = in + (size_t)row*E; + device half* pr = probs + (size_t)row*E; + device float* idx_out = tk + (size_t)row*k; + device float* val_out = tk + (size_t)B*k + (size_t)row*k; + float mx = -INFINITY; + for (uint i = lane; i < E; i += 32u) mx = max(mx, (float)(half)((float)x[i]*scale)); + mx = simd_max(mx); + float sum = 0; + for (uint i = lane; i < E; i += 32u) sum += exp((float)(half)((float)x[i]*scale) - mx); + sum = simd_sum(sum); + float inv = 1.0f/sum; + for (uint i = lane; i < E; i += 32u) pr[i] = (half)(exp((float)(half)((float)x[i]*scale) - mx)*inv); + for (uint j = 0u; j < k; ++j) { + float best = -INFINITY; uint bi = 0xFFFFFFFFu; + for (uint i = lane; i < E; i += 32u) { + bool taken = false; + for (uint p = 0u; p < j; ++p) if ((uint)idx_out[p] == i) taken = true; + float v = (float)(half)((float)x[i]*scale); + if (!taken && (v > best || (v == best && i < bi))) { best = v; bi = i; } + } + float gb = simd_max(best); + uint cand = (best == gb) ? bi : 0xFFFFFFFFu; + uint gi = simd_min(cand); + if (lane == 0u) { idx_out[j] = (float)gi; val_out[j] = gb; } + simdgroup_barrier(mem_flags::mem_device); + } +} + +kernel void rope_pair_rms_f16( + device const half* x [[buffer(0)]], + device const half* w [[buffer(1)]], + device const half* c [[buffer(2)]], + device const half* s [[buffer(3)]], + device half* y [[buffer(4)]], + constant uint& D [[buffer(5)]], + constant float& eps [[buffer(6)]], + uint h [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* red [[threadgroup(0)]], + threadgroup half* xn [[threadgroup(1)]]) +{ + device const half* xr = x + (size_t)h*D; + float ss = 0; + for (uint i = t; i < D; i += T) { float v = (float)xr[i]; ss += v*v; } + red[t] = ss; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint st = T/2; st > 0; st >>= 1) { + if (t < st) red[t] += red[t+st]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + float inv = rsqrt(red[0]/(float)D + eps); + for (uint i = t; i < D; i += T) xn[i] = (half)((float)xr[i]*inv*(float)w[i]); + threadgroup_barrier(mem_flags::mem_threadgroup); + device half* yr = y + (size_t)h*D; + for (uint i = t; i < D; i += T) { + half xv = xn[i]; + half rh = (i < D/2u) ? (half)(-xn[i + D/2u]) : xn[i - D/2u]; + yr[i] = xv*c[i] + rh*s[i]; + } +} + +kernel void rms2_add_clip_f16( + device const half* a [[buffer(0)]], + device const half* wa [[buffer(1)]], + device const half* b [[buffer(2)]], + device const half* wb [[buffer(3)]], + device half* y [[buffer(4)]], + constant uint& D [[buffer(5)]], + constant float& eps_a [[buffer(6)]], + constant float& eps_b [[buffer(7)]], + uint t [[thread_position_in_threadgroup]], + uint T [[threads_per_threadgroup]], + threadgroup float* red [[threadgroup(0)]]) +{ + float sa = 0, sb = 0; + for (uint i = t; i < D; i += T) { + float va = (float)a[i], vb = (float)b[i]; + sa += va*va; sb += vb*vb; + } + red[t] = sa; red[T+t] = sb; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint s = T/2; s > 0; s >>= 1) { + if (t < s) { red[t] += red[t+s]; red[T+t] += red[T+t+s]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + float inva = rsqrt(red[0]/(float)D + eps_a); + float invb = rsqrt(red[T]/(float)D + eps_b); + for (uint i = t; i < D; i += T) { + half ya = (half)((float)a[i]*inva*(float)wa[i]); + half yb = (half)((float)b[i]*invb*(float)wb[i]); + y[i] = (half)clamp((float)ya + (float)yb, -65500.0f, 65500.0f); + } +} + +kernel void gated_deltanet_decode_f16( + device const half* q [[buffer(0)]], + device const half* k [[buffer(1)]], + device const half* v [[buffer(2)]], + device const half* g [[buffer(3)]], + device const half* b [[buffer(4)]], + device const half* s [[buffer(5)]], + device half* y [[buffer(6)]], + constant uint& Hq [[buffer(7)]], + constant uint& Hv [[buffer(8)]], + constant uint& K [[buffer(9)]], + constant uint& V [[buffer(10)]], + constant float& scale [[buffer(11)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint t [[thread_index_in_threadgroup]], + threadgroup half* kq [[threadgroup(0)]]) +{ + const uint h = tgp.x, batch = tgp.y; + const uint qk_head = h / (Hv / Hq); + const uint qk_base = (batch * Hq + qk_head) * K; + threadgroup half* kk = kq; + threadgroup half* qq = kq + K; + for (uint i = t; i < K; i += V) { + kk[i] = k[qk_base + i]; + qq[i] = q[qk_base + i]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (t >= V) return; + + float gate_log = (float)g[batch * Hv + h]; + if (!isfinite(gate_log)) gate_log = -20.0f; + float beta = (float)b[batch * Hv + h]; + beta = isfinite(beta) ? clamp(beta, 0.0f, 1.0f) : 0.0f; + const float alpha = exp(clamp(gate_log, -20.0f, 6.0f)); + + const size_t s_stride = (size_t)Hv * V; + device const half* sp = s + ((size_t)batch * K) * s_stride + (size_t)h * V + t; + float proj = 0.0f; + for (uint kd = 0; kd < K; ++kd) proj += (float)sp[kd * s_stride] * (float)kk[kd]; + const float delta = ((float)v[((size_t)batch * Hv + h) * V + t] - alpha * proj) * beta; + + device half* yp = y + ((size_t)batch * (1 + K) + 1) * s_stride + (size_t)h * V + t; + float acc = 0.0f; + for (uint kd = 0; kd < K; ++kd) { + float s_new = (float)sp[kd * s_stride] * alpha + (float)kk[kd] * delta; + if (!isfinite(s_new)) s_new = 0.0f; + yp[kd * s_stride] = (half)s_new; + acc += s_new * (float)qq[kd]; + } + if (!isfinite(acc)) acc = 0.0f; + y[((size_t)batch * (1 + K)) * s_stride + (size_t)h * V + t] = (half)(acc * scale); +} + +kernel void gated_deltanet_prefill_f16( + device const half* q [[buffer(0)]], + device const half* k [[buffer(1)]], + device const half* v [[buffer(2)]], + device const half* g [[buffer(3)]], + device const half* b [[buffer(4)]], + device const half* s [[buffer(5)]], + device half* y [[buffer(6)]], + device float* st [[buffer(7)]], + constant uint& T [[buffer(8)]], + constant uint& Hq [[buffer(9)]], + constant uint& Hv [[buffer(10)]], + constant uint& K [[buffer(11)]], + constant uint& V [[buffer(12)]], + constant float& scale [[buffer(13)]], + uint3 tgp [[threadgroup_position_in_grid]], + uint t [[thread_index_in_threadgroup]], + threadgroup half* kq [[threadgroup(0)]]) +{ + const uint h = tgp.x, batch = tgp.y; + const uint qk_head = h / (Hv / Hq); + threadgroup half* kk = kq; + threadgroup half* qq = kq + K; + const size_t hv_stride = (size_t)Hv * V; + device float* col = st + (((size_t)batch * Hv + h) * K) * V + t; + + if (t < V) { + device const half* sp = s + ((size_t)batch * K) * hv_stride + (size_t)h * V + t; + for (uint kd = 0; kd < K; ++kd) col[(size_t)kd * V] = (float)sp[kd * hv_stride]; + } + + for (uint step = 0; step < T; ++step) { + threadgroup_barrier(mem_flags::mem_threadgroup); + const size_t qk_base = ((size_t)(batch * T + step) * Hq + qk_head) * K; + for (uint i = t; i < K; i += V) { + kk[i] = k[qk_base + i]; + qq[i] = q[qk_base + i]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (t >= V) continue; + + float gate_log = (float)g[(size_t)(batch * T + step) * Hv + h]; + if (!isfinite(gate_log)) gate_log = -20.0f; + float beta = (float)b[(size_t)(batch * T + step) * Hv + h]; + beta = isfinite(beta) ? clamp(beta, 0.0f, 1.0f) : 0.0f; + const float alpha = exp(clamp(gate_log, -20.0f, 6.0f)); + + float proj = 0.0f; + for (uint kd = 0; kd < K; ++kd) proj += col[(size_t)kd * V] * (float)kk[kd]; + const float delta = ((float)v[((size_t)(batch * T + step) * Hv + h) * V + t] - alpha * proj) * beta; + + float acc = 0.0f; + for (uint kd = 0; kd < K; ++kd) { + float s_new = col[(size_t)kd * V] * alpha + (float)kk[kd] * delta; + if (!isfinite(s_new)) s_new = 0.0f; + col[(size_t)kd * V] = s_new; + acc += s_new * (float)qq[kd]; + } + if (!isfinite(acc)) acc = 0.0f; + y[((size_t)(batch * (T + K) + step)) * hv_stride + (size_t)h * V + t] = (half)(acc * scale); + } + + if (t < V) { + for (uint kd = 0; kd < K; ++kd) + y[((size_t)(batch * (T + K) + T + kd)) * hv_stride + (size_t)h * V + t] = (half)col[(size_t)kd * V]; + } +} diff --git a/cactus-kernels/src/dsp.cpp b/cactus-kernels/src/dsp.cpp index 13d3fc6fe..bfe01eb76 100644 --- a/cactus-kernels/src/dsp.cpp +++ b/cactus-kernels/src/dsp.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -150,7 +152,19 @@ void cactus_rfft_f32_1d(const float* input, float* output, size_t n, const char* log2n++; } - FFTSetup fft_setup = vDSP_create_fftsetup(log2n, FFT_RADIX2); + static std::mutex fft_mu; + static std::unordered_map fft_cache; + FFTSetup fft_setup; + { + std::lock_guard lk(fft_mu); + auto it = fft_cache.find(log2n); + if (it == fft_cache.end()) { + FFTSetup created = vDSP_create_fftsetup(log2n, FFT_RADIX2); + if (created) it = fft_cache.emplace(log2n, created).first; + else it = fft_cache.end(); + } + fft_setup = it != fft_cache.end() ? it->second : nullptr; + } if (fft_setup) { std::vector buffer(padded_n, 0.0f); std::copy(input, input + n, buffer.begin()); @@ -182,7 +196,6 @@ void cactus_rfft_f32_1d(const float* input, float* output, size_t n, const char* output[i * 2 + 1] = split.imagp[i] * scale; } - vDSP_destroy_fftsetup(fft_setup); return; } } diff --git a/cactus-kernels/src/matmul.cpp b/cactus-kernels/src/matmul.cpp index a082d3c91..2873ac911 100644 --- a/cactus-kernels/src/matmul.cpp +++ b/cactus-kernels/src/matmul.cpp @@ -1749,24 +1749,37 @@ void cactus_quant_matmul( constexpr size_t TILE_M = 8; + struct ExpandEntry { + uint64_t fingerprint = 0; + std::vector weights; + std::vector norms; + }; static std::mutex s_expand_mutex; - static std::unordered_map, std::vector>> s_expand_cache; + static std::unordered_map s_expand_cache; + const size_t packed_bytes = static_cast(W->N) * num_groups * pgb; + uint64_t fp = 1469598103934665603ull; + auto fp_mix = [&fp](uint64_t v) { fp ^= v; fp *= 1099511628211ull; }; + fp_mix(bits); fp_mix(W->K); fp_mix(W->N); fp_mix(gs); + const uint8_t* pb = static_cast(W->packed_indices); + for (size_t i = 0; i < 64 && i < packed_bytes; ++i) fp_mix(pb[i]); + for (size_t i = packed_bytes > 64 ? packed_bytes - 64 : 0; i < packed_bytes; ++i) fp_mix(pb[i]); const int8_t* w_il; const float* n_f32; { std::lock_guard lock(s_expand_mutex); auto& entry = s_expand_cache[W->packed_indices]; - if (entry.first.empty()) { + if (entry.weights.empty() || entry.fingerprint != fp) { int8_t cb_i8[16] = {}; const float cb_scale = tq_quantize_codebook_i8(W->codebook, cb_i8, codebook_size); const int8x16_t cb_lut = vld1q_s8(cb_i8); - entry.first.resize(expanded_size); - entry.second.resize(expanded_norms_size); + entry.weights.assign(expanded_size, 0); + entry.norms.assign(expanded_norms_size, 0.0f); tq_preexpand_weights(W, bits, num_groups, gs, pgb, cb_lut, cb_scale, - N_blocks, entry.first.data(), entry.second.data()); + N_blocks, entry.weights.data(), entry.norms.data()); + entry.fingerprint = fp; } - w_il = entry.first.data(); - n_f32 = entry.second.data(); + w_il = entry.weights.data(); + n_f32 = entry.norms.data(); } const size_t M_blocks = (M + TILE_M - 1) / TILE_M; diff --git a/cactus-kernels/src/metal_backend.mm b/cactus-kernels/src/metal_backend.mm new file mode 100644 index 000000000..0c4a5f236 --- /dev/null +++ b/cactus-kernels/src/metal_backend.mm @@ -0,0 +1,2489 @@ + + +#include "metal_backend.h" + +#if CACTUS_HAS_METAL + +#import +#import +#include +#include +#if TARGET_OS_OSX +#include +#endif +#import +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cactus_kernels_msl.h" +#include "threading.h" +#include + +namespace { + +struct MetalCtx { + id dev = nil; + id queue = nil; + id psoT = nil; + id psoTsimd = nil; + id psoTbatch = nil; + id psoMega = nil, psoSwiT = nil, psoCat = nil; + id psoG = nil, psoGmr = nil, psoGmrLow = nil; + id psoActQ = nil, psoG2i8 = nil; + id psoTm = nil; + id psoGmma = nil, psoGdense = nil; + id psoRotW = nil, psoEmbOW = nil, psoRmsAddRms = nil; + id psoEmbH = nil; + id psoEmbOm = nil, psoEmbHm = nil; + id psoCopy=nil, psoBinary=nil, psoScalar=nil, psoUnary=nil, psoRms=nil, psoSwiglu=nil, psoRmsAdd=nil, psoRmsAddScale=nil; + id psoCF16F32=nil, psoCF32F16=nil, psoCI8F16=nil, psoCF16I8=nil; + id psoAttn=nil, psoAttnC=nil, psoAttnF=nil, psoStrided=nil, psoScatter=nil, psoBcast=nil, psoKvAppend=nil; + id psoAttnPre=nil, psoAttnPreMma2=nil, psoAttnPreHd256=nil; + id psoKvAppendM=nil, psoKvAppendRingM=nil; + id psoSlideS=nil, psoSlideR=nil, psoSlideRM=nil; + id psoArgmax=nil; + id psoArgmaxP=nil, psoArgmaxF=nil, psoSoftcap=nil, psoAdjust=nil; + id psoGather=nil; + id psoBinF32=nil, psoScaF32=nil, psoUnaF32=nil; + id psoClampF16=nil, psoClampF32=nil, psoGlu=nil; + id psoLayerNorm=nil, psoSoftmaxR=nil, psoConv1dK3=nil, psoAttnDense=nil, psoAttnD64=nil; + id psoGemmBatch=nil, psoGemmBatchF32A=nil, psoConvDw=nil, psoStridedRows=nil, psoMaskFill=nil; + id psoReduceF16=nil, psoReduceF32=nil, psoCumsumF16=nil, psoCumsumF32=nil; + id psoConcat2=nil, psoGatherF32Idx=nil, psoRopeFull=nil, psoMaxpool1d=nil; + id psoBilinear=nil, psoConv1dGen=nil, psoConv1dNlcDw=nil, psoConv2d=nil; + id psoBatchnorm=nil, psoGroupnorm=nil, psoBiasAddRows=nil, psoEwChain=nil; + id psoAttnFlash=nil, psoRmsSimd=nil, psoScatterRows=nil; + id psoTranspose2d=nil, psoBcastRows=nil, psoRmsAddSimd=nil; + id psoConvCacheAppend=nil, psoRelPosBias=nil, psoGemvBias=nil; + id psoTopkRows=nil, psoMoeT=nil, psoMoeUp=nil; + id psoMoeT2=nil, psoMoeDownAcc=nil, psoRopePair=nil; + id psoRmsScale=nil, psoSoftmaxTopk=nil, psoRopePairRms=nil; + id psoRms2AddClip=nil, psoDeltanet=nil, psoDeltanetPre=nil; + id dummy=nil; + bool ok = false; + + MetalCtx() { @autoreleasepool { + dev = MTLCreateSystemDefaultDevice(); + if (!dev) return; + queue = [dev newCommandQueue]; + NSError* err = nil; + id lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kCactusMSL] + options:nil error:&err]; + if (!lib) { if (err) fprintf(stderr,"[cactus-metal] MSL compile failed: %s\n",[[err localizedDescription] UTF8String]); return; } + auto pso = [&](const char* name) -> id { + NSError* e=nil; + id f=[lib newFunctionWithName:[NSString stringWithUTF8String:name]]; + id p = f ? [dev newComputePipelineStateWithFunction:f error:&e] : nil; + if (!p) fprintf(stderr,"[cactus-metal] pipeline '%s' failed: %s\n", name, e?[[e localizedDescription] UTF8String]:"function not found"); + return p; + }; + psoT=pso("cq4_transform"); psoTsimd=pso("cq4_transform_simd"); psoG=pso("cq4_gemv"); psoTbatch=pso("cq4_transform_batch"); psoGmr=pso("cq4_gemv_mr"); + psoMega=pso("cq4_transform_gemv"); psoSwiT=pso("cq4_swiglu_transform"); psoCat=pso("cq4_gemv_cat"); + psoTm=pso("cq4_transform_m"); psoGmma=pso("cq4_gemm_mma"); + psoGdense=pso("cq4_gemm_dense_f16"); + psoGmrLow=pso("cq_gemv_mr_lowbit"); + psoActQ=pso("cq_act_quant_i8"); psoG2i8=pso("cq2_gemv_i8"); + psoRotW=pso("lmhead_rotate_wide"); psoEmbOW=pso("emb_ortho_wide"); psoRmsAddRms=pso("rms_norm_add_rms_f16"); + psoEmbH=pso("emb_hadamard"); + psoEmbOm=pso("emb_ortho_m"); psoEmbHm=pso("emb_hadamard_m"); + psoGather=pso("gather_f16"); + psoCopy=pso("copy_bytes"); psoBinary=pso("binary_f16"); psoScalar=pso("scalar_f16"); + psoUnary=pso("unary_f16"); psoRms=pso("rms_norm_f16"); psoSwiglu=pso("swiglu_f16"); psoRmsAdd=pso("rms_norm_add_f16"); psoRmsAddScale=pso("rms_norm_add_scale_f16"); + psoCF16F32=pso("cast_f16_f32"); psoCF32F16=pso("cast_f32_f16"); + psoCI8F16=pso("cast_i8_f16"); psoCF16I8=pso("cast_f16_i8"); + psoAttn=pso("attn_decode_i8"); psoAttnC=pso("attn_decode_combine"); psoAttnF=pso("attn_decode_fused_i8"); + psoStrided=pso("strided_copy_f16"); psoBcast=pso("bcast_binary_f16"); + psoAttnPre=pso("attn_prefill_i8"); psoAttnPreMma2=pso("attn_prefill_mma2"); psoAttnPreHd256=pso("attn_prefill_mma_hd256"); psoKvAppendM=pso("kv_append_i8_m"); + psoKvAppendRingM=pso("kv_append_ring_i8_m"); + psoSlideS=pso("kv_slide_save"); psoSlideR=pso("kv_slide_restore"); psoSlideRM=pso("kv_slide_restore_m"); + psoScatter=pso("strided_scatter_f16"); psoKvAppend=pso("kv_append_i8"); + psoArgmax=pso("argmax_logits"); + psoArgmaxP=pso("argmax_part"); psoArgmaxF=pso("argmax_final"); psoSoftcap=pso("softcap_f16"); + psoAdjust=pso("adjust_logits"); + psoBinF32=pso("binary_f32"); psoScaF32=pso("scalar_f32"); psoUnaF32=pso("unary_f32"); + psoClampF16=pso("clamp_f16"); psoClampF32=pso("clamp_f32"); psoGlu=pso("glu_f16"); + psoLayerNorm=pso("layer_norm_f16"); psoSoftmaxR=pso("softmax_rows_f16"); + psoConv1dK3=pso("conv1d_k3_f16"); psoAttnDense=pso("attn_f16"); psoAttnD64=pso("attn_f16_d64"); + psoGemmBatch=pso("gemm_batch_f16"); psoGemmBatchF32A=pso("gemm_batch_f32a"); psoConvDw=pso("conv1d_dw_f16"); + psoStridedRows=pso("strided_copy_rows_f16"); psoMaskFill=pso("attn_maskfill_f16"); + psoReduceF16=pso("reduce_axis_f16"); psoReduceF32=pso("reduce_axis_f32"); + psoCumsumF16=pso("cumsum_f16"); psoCumsumF32=pso("cumsum_f32"); + psoConcat2=pso("concat2_f16"); psoGatherF32Idx=pso("gather_f32idx_f16"); + psoRopeFull=pso("rope_full_f16"); psoMaxpool1d=pso("maxpool1d_f16"); + psoBilinear=pso("bilinear_f16"); psoConv1dGen=pso("conv1d_gen_f16"); + psoConv1dNlcDw=pso("conv1d_nlc_dw_f16"); psoConv2d=pso("conv2d_f16"); + psoBatchnorm=pso("batchnorm_f16"); psoGroupnorm=pso("groupnorm_f16"); + psoBiasAddRows=pso("bias_add_rows_f16"); psoEwChain=pso("elemwise_chain_f16"); + psoAttnFlash=pso("attn_flash_f16"); psoRmsSimd=pso("rms_norm_simd_f16"); + psoScatterRows=pso("strided_scatter_rows_f16"); psoTranspose2d=pso("transpose2d_f16"); + psoBcastRows=pso("bcast_binary_rows_f16"); psoRmsAddSimd=pso("rms_norm_add_simd_f16"); + psoConvCacheAppend=pso("conv_cache_append_f16"); psoRelPosBias=pso("rel_pos_bias_f16"); + psoGemvBias=pso("gemv_bias_f16"); + psoTopkRows=pso("topk_rows_f16"); psoMoeT=pso("cq4_moe_transform"); + psoMoeUp=pso("cq4_moe_gemv_up"); psoMoeT2=pso("cq4_moe_transform2"); + psoMoeDownAcc=pso("cq4_moe_gemv_down_acc"); psoRopePair=pso("rope_pair_f16"); + psoRmsScale=pso("rms_norm_scale_f16"); psoSoftmaxTopk=pso("softmax_topk_f16"); + psoRopePairRms=pso("rope_pair_rms_f16"); psoRms2AddClip=pso("rms2_add_clip_f16"); + psoDeltanet=pso("gated_deltanet_decode_f16"); psoDeltanetPre=pso("gated_deltanet_prefill_f16"); + dummy=[dev newBufferWithLength:16 options:MTLResourceStorageModeShared]; + ok = psoT&&psoG&&psoTm&&psoGmma&&psoRotW&&psoEmbH&&psoEmbOm&&psoEmbHm&&psoCopy&&psoBinary&&psoScalar&&psoUnary&&psoRms&&psoSwiglu&&psoRmsAdd&&psoCF16F32&&psoCF32F16&&psoCI8F16&&psoCF16I8 + &&psoAttn&&psoAttnC&&psoAttnPre&&psoAttnPreMma2&&psoKvAppendM&&psoKvAppendRingM&&psoSlideS&&psoSlideR&&psoSlideRM&&psoStrided&&psoBcast&&psoScatter&&psoKvAppend&&psoArgmax&&psoGather; + }} +}; + +MetalCtx& ctx() { static MetalCtx c; return c; } + +id owned_shared(size_t len) { + size_t alen = ((len ? len : 1) + 16383) & ~(size_t)16383; + void* p = mmap(nullptr, alen, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); + if (p == MAP_FAILED) + return [ctx().dev newBufferWithLength:alen options:MTLResourceStorageModeShared]; + id b = [ctx().dev newBufferWithBytesNoCopy:p length:alen + options:MTLResourceStorageModeShared + deallocator:^(void* ptr, NSUInteger l){ munmap(ptr, l); }]; + if (!b) { + munmap(p, alen); + return [ctx().dev newBufferWithLength:alen options:MTLResourceStorageModeShared]; + } + return b; +} + +inline id buf(const void* p, size_t bytes) { + if (!p || bytes == 0) return nil; + id b = owned_shared(bytes); + if (b) std::memcpy([b contents], p, bytes); + return b; +} + +struct ResW { + id packed=nil, norms=nil, codebook=nil, lsign=nil, rsign=nil, perm=nil, recip=nil; + id rotation=nil; + id cb_i8=nil; + float cb_scale=1.f; + size_t packed_off=0, norms_off=0; + size_t cb_off=0, ls_off=0, rs_off=0, pm_off=0, rc_off=0, rot_off=0; + uint32_t il=0; + bool ok=false; +}; +std::unordered_map g_resident; + +static uint64_t resident_key(const CactusQuantMatrix* W) { + uint64_t h = 1469598103934665603ull; + auto mix = [&](uint64_t v) { h ^= v; h *= 1099511628211ull; }; + mix(W->bits); mix(W->K); mix(W->N); mix(W->group_size); mix(W->num_groups); mix(W->flags); + const uint8_t* p = (const uint8_t*)W->packed_indices; + size_t pkb = (size_t)W->N * W->num_groups * ((W->group_size * W->bits + 7u) / 8u); + if (!p || pkb == 0) return h; + size_t take = pkb < 64 ? pkb : 64; + for (size_t i = 0; i < take; ++i) mix(p[i]); + for (size_t i = pkb - take; i < pkb; ++i) mix(p[i]); + return h; +} +id g_attn_scores = nil; +std::unordered_map g_resident_emb; +std::mutex g_resident_mu; + +struct MoESet4 { + id pk=nil, nm=nil, rc=nil, ls=nil, rs=nil, pm=nil, cb=nil; + size_t pk_off=0, nm_off=0, rc_off=0, ls_off=0, rs_off=0, pm_off=0, cb_off=0; + uint32_t K=0, N=0; + uint32_t stride=0; +}; +struct MoECat4 { MoESet4 w1, w3, w2; bool ok=false; }; +std::unordered_map g_moe_cat; + +std::pair, size_t> wrapHostPtr(const void* p, size_t bytes); + +bool bind_component(const void* p, size_t bytes, size_t align, + id __strong& b, size_t& off) { + if (!p || bytes == 0) return true; + if (((uintptr_t)p & (align-1)) != 0) return false; + auto w = wrapHostPtr(p, bytes); + if (!w.first) return false; + b = w.first; + off = w.second; + return true; +} + +ResW& resident(const CactusQuantMatrix* W) { + std::lock_guard lk(g_resident_mu); + uint64_t key = resident_key(W); + auto it = g_resident.find(key); + if (it != g_resident.end()) return it->second; + const uint32_t K=W->K, N=W->N, ng=W->num_groups, gs=W->group_size, bits=W->bits; + const uint32_t pgb=(gs*bits+7u)/8u; + ResW r; + r.ok = bind_component(W->codebook, (size_t)(1u<left_signs, gs, 1, r.lsign, r.ls_off) + && bind_component(W->right_signs, gs, 1, r.rsign, r.rs_off) + && bind_component(W->permutation, (size_t)gs*sizeof(uint32_t), 4, r.perm, r.pm_off) + && bind_component(W->input_scale_recip, (size_t)K*sizeof(__fp16), 2, r.recip, r.rc_off) + && (!W->rotation || bind_component(W->rotation, (size_t)K*K*sizeof(__fp16), 8, r.rotation, r.rot_off)); + if (bits == 2u) { + int8_t cbq[4]; + float max_abs = 0.f; + for (uint32_t i = 0; i < 4u; i++) { + float v = std::fabs((float)W->codebook[i]); + if (v > max_abs) max_abs = v; + } + float scale = max_abs / 127.f; + if (scale < 1e-10f) scale = 1e-10f; + float inv = 1.f / scale; + for (uint32_t i = 0; i < 4u; i++) + cbq[i] = (int8_t)std::round((float)W->codebook[i] * inv); + r.cb_i8 = buf(cbq, 4); + r.cb_scale = scale; + } + size_t pkb=(size_t)N*ng*pgb, nmb=(size_t)N*ng*sizeof(__fp16); + const bool interleaved = (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) != 0; + if (((uintptr_t)W->packed_indices & 7u) != 0 || ((uintptr_t)W->norms & 1u) != 0) { + r.ok = false; + } else if (r.ok) { + auto wp = wrapHostPtr(W->packed_indices, pkb); + auto wn = wrapHostPtr(W->norms, nmb); + if (wp.first && wn.first) { + r.packed = wp.first; r.packed_off = wp.second; + r.norms = wn.first; r.norms_off = wn.second; + r.il = interleaved ? 1 : 0; + } else { + r.ok = false; + } + } + return g_resident.emplace(key, r).first->second; +} + +id g_cmd = nil; +id g_enc = nil; +std::unordered_map>> g_free; +std::vector> g_pending; +std::map> g_shared; +bool g_active = false; +id g_code_buf = nil; +id g_code_buf_m = nil; + +inline size_t bucket(size_t b) { return (b + 16383) & ~size_t(16383); } + +static const uint32_t g_mr_rows = 8u * 2u; + +inline bool tg_mem_ok(size_t bytes) { + return bytes <= (size_t)[ctx().dev maxThreadgroupMemoryLength]; +} + +id ensureEncoder() { + if (!g_enc) { + if (!g_cmd) g_cmd = [ctx().queue commandBuffer]; + g_enc = [g_cmd computeCommandEncoder]; + } + return g_enc; +} + +id recycled(size_t bytes) { + size_t bk = bucket(bytes ? bytes : 1); + auto it = g_free.find(bk); + id b; + if (it != g_free.end() && !it->second.empty()) { b = it->second.back(); it->second.pop_back(); } + else b = owned_shared(bk); + g_pending.push_back(b); + return b; +} + +std::map> g_wrapped; +std::unordered_map> g_readonly; +std::map, MPSMatrix*> g_mps_mats; +std::pair, size_t> wrapHostPtr(const void* p, size_t bytes) { + uintptr_t a = (uintptr_t)p, base = a & ~(uintptr_t)16383u; + size_t need = (a - base) + (bytes ? bytes : 1); + auto it = g_wrapped.upper_bound(a); + if (it != g_wrapped.begin()) { + --it; + if (a >= it->first && a + (bytes?bytes:1) <= it->first + (size_t)it->second.length) + return { it->second, (size_t)(a - it->first) }; + } + size_t wraplen = (need + 16383u) & ~(size_t)16383u; + uintptr_t lo = base, hi = base + wraplen; +#if TARGET_OS_OSX + // Expand the wrap to the whole VM region so neighboring host pointers hit + // the same MTLBuffer. mach_vm.h is macOS-only; elsewhere the page-aligned + // span above is used as-is (correct, just more wrap entries). + { + mach_vm_address_t raddr = a; + mach_vm_size_t rsize = 0; + vm_region_basic_info_data_64_t info; + mach_msg_type_number_t cnt = VM_REGION_BASIC_INFO_COUNT_64; + mach_port_t obj = MACH_PORT_NULL; + if (mach_vm_region(mach_task_self(), &raddr, &rsize, VM_REGION_BASIC_INFO_64, + (vm_region_info_t)&info, &cnt, &obj) == KERN_SUCCESS + && raddr <= base && raddr + rsize >= hi + && (info.protection & VM_PROT_READ) && (info.protection & VM_PROT_WRITE) + && rsize <= (512u << 20)) { + lo = (uintptr_t)raddr; + hi = (uintptr_t)(raddr + rsize); + } + if (obj != MACH_PORT_NULL) mach_port_deallocate(mach_task_self(), obj); + } +#endif + auto ov = g_wrapped.lower_bound(lo); + if (ov != g_wrapped.begin()) { + auto prev = std::prev(ov); + if (prev->first + (size_t)prev->second.length > lo) ov = prev; + } + while (ov != g_wrapped.end() && ov->first < hi) { + if (ov->first + (size_t)ov->second.length > lo) { + if (ov->first < lo) lo = ov->first; + if (ov->first + (size_t)ov->second.length > hi) hi = ov->first + (size_t)ov->second.length; + ov = g_wrapped.erase(ov); + } else ++ov; + } + id b = [ctx().dev newBufferWithBytesNoCopy:(void*)lo length:(size_t)(hi - lo) + options:MTLResourceStorageModeShared deallocator:nil]; + if (!b) return { nil, 0 }; + g_wrapped[lo] = b; + return { b, (size_t)(a - lo) }; +} + +std::pair, size_t> bufForPtrOff(const void* p, size_t bytes) { + uintptr_t a = reinterpret_cast(p); + auto it = g_shared.upper_bound(a); + if (it != g_shared.begin()) { + --it; + uintptr_t base = it->first; + if (a < base + static_cast(it->second.length)) + return { it->second, static_cast(a - base) }; + } + + auto wr = wrapHostPtr(p, bytes); + if (wr.first) return wr; + + size_t nb = bytes ? bytes : 1; + id b = recycled(bytes); + std::memcpy([b contents], p, nb); + return { b, 0 }; +} + +inline void setBufAt(const void* p, size_t bytes, int idx) { + auto pr = bufForPtrOff(p, bytes); + [g_enc setBuffer:pr.first offset:pr.second atIndex:idx]; +} + +} + +bool cactus_metal_available() { return ctx().ok; } + +void cactus_metal_set_active(bool a) { g_active = a; } +bool cactus_metal_active_mode() { return ctx().ok && g_active; } + +extern "C" void* objc_autoreleasePoolPush(void); +extern "C" void objc_autoreleasePoolPop(void*); +static void* g_arpool = nullptr; + +void cactus_metal_session_begin() { + if (!g_arpool) g_arpool = objc_autoreleasePoolPush(); +} +static id g_last_cmd = nil; +static std::atomic g_cmd_failed{false}; +static void watch_cmd_errors(id cmd) { + [cmd addCompletedHandler:^(id cb) { + if (cb.status == MTLCommandBufferStatusError) { + g_cmd_failed.store(true, std::memory_order_relaxed); + fprintf(stderr, "[cactus-metal] command buffer failed: %s\n", + cb.error ? [[cb.error localizedDescription] UTF8String] : "unknown"); + } + }]; +} +static void reclaim_dead_slabs(); +void cactus_metal_session_sync() { + if (g_enc) { [g_enc endEncoding]; g_enc = nil; } + if (g_cmd) { + watch_cmd_errors(g_cmd); + [g_cmd commit]; + g_last_cmd = g_cmd; + g_cmd = nil; + } + if (g_last_cmd) { + [g_last_cmd waitUntilCompleted]; + g_last_cmd = nil; + } + for (id b : g_pending) g_free[(size_t)b.length].push_back(b); + g_pending.clear(); + reclaim_dead_slabs(); +} +void cactus_metal_session_end() { + cactus_metal_session_sync(); + if (g_arpool) { objc_autoreleasePoolPop(g_arpool); g_arpool = nullptr; } +} +void cactus_metal_invalidate_host_wraps() { + std::lock_guard lk(g_resident_mu); + g_attn_scores = nil; + g_wrapped.clear(); + g_readonly.clear(); + g_resident.clear(); + g_resident_emb.clear(); + g_mps_mats.clear(); + g_moe_cat.clear(); +} + +void cactus_metal_session_flush() { + if (g_enc) { [g_enc endEncoding]; g_enc = nil; } + if (g_cmd) { watch_cmd_errors(g_cmd); [g_cmd commit]; g_last_cmd = g_cmd; g_cmd = nil; } +} + + +void* cactus_metal_alloc_shared(size_t bytes) { + if (!ctx().ok) return nullptr; + size_t bk = bucket(bytes ? bytes : 1); + id b = nil; + auto it = g_free.find(bk); + if (it != g_free.end() && !it->second.empty()) { b = it->second.back(); it->second.pop_back(); } + else b = owned_shared(bk); + void* c = [b contents]; + g_shared[reinterpret_cast(c)] = b; + return c; +} + +struct SlabInfo { size_t cap = 0, used = 0, live = 0; }; +static char* g_slab = nullptr; +static std::map g_slabs; + +static void reclaim_dead_slabs() { + for (auto it = g_slabs.begin(); it != g_slabs.end();) { + SlabInfo& s = it->second; + if (s.live != 0 || s.used == 0) { ++it; continue; } + if ((char*)it->first == g_slab) { s.used = 0; ++it; continue; } + g_shared.erase(it->first); + it = g_slabs.erase(it); + } +} + +void* cactus_metal_alloc_pooled(size_t bytes) { + if (!ctx().ok) return nullptr; + if (bytes > (4u << 20)) return cactus_metal_alloc_shared(bytes); + size_t need = (bytes + 255) & ~size_t(255); + SlabInfo* cur = nullptr; + if (g_slab) { + auto it = g_slabs.find(reinterpret_cast(g_slab)); + if (it != g_slabs.end()) cur = &it->second; + } + if (!cur || cur->used + need > cur->cap) { + size_t cap = 32u << 20; + id b = owned_shared(cap); + if (!b) return cactus_metal_alloc_shared(bytes); + g_slab = (char*)[b contents]; + g_shared[reinterpret_cast(g_slab)] = b; + cur = &g_slabs[reinterpret_cast(g_slab)]; + *cur = SlabInfo{cap, 0, 0}; + } + void* p = g_slab + cur->used; + cur->used += need; + cur->live += 1; + return p; +} +void cactus_metal_free_shared(void* contents) { + uintptr_t a = reinterpret_cast(contents); + auto sit = g_slabs.upper_bound(a); + if (sit != g_slabs.begin()) { + --sit; + if (a < sit->first + sit->second.cap) { + if (sit->second.live > 0) --sit->second.live; + return; + } + } + auto it = g_shared.find(a); + if (it != g_shared.end()) { g_pending.push_back(it->second); g_shared.erase(it); } +} + +bool cactus_metal_encode_copy(void* out, const void* in, size_t bytes) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t n=(uint32_t)bytes; + [g_enc setComputePipelineState:ctx().psoCopy]; + setBufAt(in, bytes, 0); + setBufAt(out, bytes, 1); + [g_enc setBytes:&n length:4 atIndex:2]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_binary(int op, void* out, const void* a, const void* b, size_t n) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; + [g_enc setComputePipelineState:ctx().psoBinary]; + setBufAt(a, n*2, 0); setBufAt(b, n*2, 1); setBufAt(out, n*2, 2); + [g_enc setBytes:&nn length:4 atIndex:3]; [g_enc setBytes:&o length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake((nn+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_scalar(int op, void* out, const void* in, size_t n, float param) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; float p=param; + [g_enc setComputePipelineState:ctx().psoScalar]; + setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; [g_enc setBytes:&p length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake((nn+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_unary(int op, void* out, const void* in, size_t n) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; + [g_enc setComputePipelineState:ctx().psoUnary]; + setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; + [g_enc dispatchThreads:MTLSizeMake((nn+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_swiglu(void* out, const void* gate, const void* up, size_t n, float scale) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; float s=scale; + [g_enc setComputePipelineState:ctx().psoSwiglu]; + setBufAt(gate, n*2, 0); setBufAt(up, n*2, 1); setBufAt(out, n*2, 2); + [g_enc setBytes:&nn length:4 atIndex:3]; [g_enc setBytes:&s length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_rms_norm(void* out, const void* in, const void* weight, + size_t rows, size_t dim, float eps) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t d=(uint32_t)dim; float e=eps; + if (rows > 1 && ctx().psoRmsSimd) { + uint32_t r=(uint32_t)rows; + [g_enc setComputePipelineState:ctx().psoRmsSimd]; + setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(out, rows*dim*2, 2); + [g_enc setBytes:&d length:4 atIndex:3]; [g_enc setBytes:&e length:4 atIndex:4]; + [g_enc setBytes:&r length:4 atIndex:5]; + [g_enc dispatchThreadgroups:MTLSizeMake((rows+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + return true; + } + [g_enc setComputePipelineState:ctx().psoRms]; + setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(out, rows*dim*2, 2); + [g_enc setBytes:&d length:4 atIndex:3]; [g_enc setBytes:&e length:4 atIndex:4]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_rms_norm_add(void* out, const void* in, const void* weight, const void* res, + size_t rows, size_t dim, float eps) { + if (!ctx().ok) return false; + ensureEncoder(); + uint32_t d=(uint32_t)dim; float e=eps; + [g_enc setComputePipelineState:ctx().psoRmsAdd]; + setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(res, rows*dim*2, 2); setBufAt(out, rows*dim*2, 3); + [g_enc setBytes:&d length:4 atIndex:4]; [g_enc setBytes:&e length:4 atIndex:5]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_rms_norm_add_scale(void* out, const void* in, const void* weight, const void* res, + size_t rows, size_t dim, float eps, float out_scale) { + if (!ctx().ok || !ctx().psoRmsAddScale) return false; + ensureEncoder(); + uint32_t d=(uint32_t)dim; float e=eps, os=out_scale; + [g_enc setComputePipelineState:ctx().psoRmsAddScale]; + setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(res, rows*dim*2, 2); setBufAt(out, rows*dim*2, 3); + [g_enc setBytes:&d length:4 atIndex:4]; [g_enc setBytes:&e length:4 atIndex:5]; [g_enc setBytes:&os length:4 atIndex:6]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_rms_norm_add_rms(void* h_out, void* xn_out, const void* in, const void* w1, + const void* res, const void* w2, + size_t rows, size_t dim, float eps, float out_scale) { + if (!ctx().ok || !ctx().psoRmsAddRms) return false; + ensureEncoder(); + uint32_t d=(uint32_t)dim; float e=eps, os=out_scale; + [g_enc setComputePipelineState:ctx().psoRmsAddRms]; + setBufAt(in, rows*dim*2, 0); setBufAt(w1, dim*2, 1); setBufAt(res, rows*dim*2, 2); + setBufAt(h_out, rows*dim*2, 3); setBufAt(w2, dim*2, 4); setBufAt(xn_out, rows*dim*2, 5); + [g_enc setBytes:&d length:4 atIndex:6]; [g_enc setBytes:&e length:4 atIndex:7]; [g_enc setBytes:&os length:4 atIndex:8]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} + +bool cactus_metal_encode_argmax(const void* logits, uint32_t vocab, void* out3, const void* bias) { + if (!ctx().ok) return false; + if (bias && !(ctx().psoArgmaxP && ctx().psoArgmaxF && vocab >= 32768u)) return false; + ensureEncoder(); + uint32_t V = vocab; + if (ctx().psoArgmaxP && ctx().psoArgmaxF && V >= 32768u) { + const uint32_t NP = 128u, T = 256u; + uint32_t chunk = (V + NP - 1u)/NP; + uint32_t has_bias = bias ? 1u : 0u; + id part = recycled((size_t)NP*3*sizeof(float)); + [g_enc setComputePipelineState:ctx().psoArgmaxP]; + setBufAt(logits, (size_t)vocab*2, 0); + [g_enc setBuffer:part offset:0 atIndex:1]; + [g_enc setBytes:&V length:4 atIndex:2]; [g_enc setBytes:&chunk length:4 atIndex:3]; + if (bias) setBufAt(bias, (size_t)vocab*sizeof(float), 4); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:4]; + [g_enc setBytes:&has_bias length:4 atIndex:5]; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:T*sizeof(uint) atIndex:1]; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:2]; + [g_enc dispatchThreadgroups:MTLSizeMake(NP,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + [g_enc setComputePipelineState:ctx().psoArgmaxF]; + [g_enc setBuffer:part offset:0 atIndex:0]; + setBufAt(out3, 3*sizeof(float), 1); + [g_enc setBytes:&NP length:4 atIndex:2]; + [g_enc setThreadgroupMemoryLength:NP*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:NP*sizeof(uint) atIndex:1]; + [g_enc setThreadgroupMemoryLength:NP*sizeof(float) atIndex:2]; + [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(NP,1,1)]; + return true; + } + uint32_t T = 1024; + [g_enc setComputePipelineState:ctx().psoArgmax]; + setBufAt(logits, (size_t)vocab*2, 0); + setBufAt(out3, 3*sizeof(float), 1); + [g_enc setBytes:&V length:4 atIndex:2]; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:T*sizeof(uint) atIndex:1]; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:2]; + [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + return true; +} +bool cactus_metal_encode_adjust_logits(void* logits, size_t vocab, + const uint32_t* recent, uint32_t n_recent, + int64_t suppressed, float penalty) { + if (!ctx().ok || !ctx().psoAdjust) return false; + if (n_recent > 4096u) return false; + ensureEncoder(); + id rb = ctx().dummy; + if (n_recent) { rb = recycled((size_t)n_recent*4); std::memcpy([rb contents], recent, (size_t)n_recent*4); } + struct { uint32_t n_recent, suppress_flag, suppress_id, vocab_n; float penalty; } U = + { n_recent, + (uint32_t)(suppressed >= 0 ? 1 : 0), (uint32_t)(suppressed >= 0 ? suppressed : 0), + (uint32_t)vocab, penalty }; + [g_enc setComputePipelineState:ctx().psoAdjust]; + setBufAt(logits, vocab*2, 0); + [g_enc setBuffer:rb offset:0 atIndex:1]; + [g_enc setBytes:&U length:sizeof(U) atIndex:2]; + [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(1024,1,1)]; + return true; +} + +bool cactus_metal_encode_softcap(void* out, const void* in, size_t n, float cap) { + if (!ctx().ok || !ctx().psoSoftcap) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; float c=cap; + [g_enc setComputePipelineState:ctx().psoSoftcap]; + setBufAt(in, n*2, 0); setBufAt(out, n*2, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&c length:4 atIndex:3]; + [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_cast(void* out, int out_prec, const void* in, int in_prec, size_t n) { + if (!ctx().ok) return false; + + auto esz=[](int p){ return p==0?1: p==1?2: p==2?4: 0; }; + if (in_prec==out_prec) return cactus_metal_encode_copy(out, in, n*esz(in_prec)); + id pso=nil; + if (in_prec==1&&out_prec==2) pso=ctx().psoCF16F32; + else if (in_prec==2&&out_prec==1) pso=ctx().psoCF32F16; + else if (in_prec==0&&out_prec==1) pso=ctx().psoCI8F16; + else if (in_prec==1&&out_prec==0) pso=ctx().psoCF16I8; + else return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; + [g_enc setComputePipelineState:pso]; + setBufAt(in, n*esz(in_prec), 0); setBufAt(out, n*esz(out_prec), 1); + [g_enc setBytes:&nn length:4 atIndex:2]; + [g_enc dispatchThreads:MTLSizeMake(nn,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +static inline bool quant_fast_eligible(const CactusQuantMatrix* W) { + return W->bits == 4 && W->group_size >= 128 && (W->group_size % 128) == 0 && (W->N % 4) == 0 && + (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) && + !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->input_scale_recip && W->left_signs && + W->right_signs && W->permutation && W->codebook && W->norms && W->packed_indices; +} + +static inline bool quant_lowbit_eligible(const CactusQuantMatrix* W) { + return (W->bits == 2 || W->bits == 3) && W->group_size == 128 && (W->N % 16) == 0 && + (W->flags & CACTUS_QUANT_FLAG_INTERLEAVED_4ROW) && + !(W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && + W->input_scale_recip && W->left_signs && W->right_signs && W->permutation && + W->codebook && W->norms && W->packed_indices; +} + +void cactus_metal_trim_prefill_cache() { + std::lock_guard lk(g_resident_mu); + g_mps_mats.clear(); + g_free.clear(); + g_attn_scores = nil; + g_code_buf = nil; + g_code_buf_m = nil; +} + +bool cactus_metal_encode_quant_matmul(void* out, const void* lhs, const CactusQuantMatrix* W) { + if (!ctx().ok || !W) return false; + const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; + if ((W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL) && W->rotation && ng==1 && gs==K && (N%4)==0) { + static void* ortho_code = nullptr; static uint32_t ortho_code_k = 0; + if (ortho_code_k < K) { + if (ortho_code) cactus_metal_free_shared(ortho_code); + ortho_code = cactus_metal_alloc_shared((size_t)K*sizeof(__fp16)); + ortho_code_k = K; + } + if (ortho_code) return cactus_metal_encode_quant_matmul_ortho(out, lhs, ortho_code, W); + } + const bool fast = quant_fast_eligible(W); + const bool lowbit = !fast && ctx().psoGmrLow && quant_lowbit_eligible(W); + if (!fast && !lowbit) return false; + const uint32_t bits = W->bits, pgbw = (gs*bits+7u)/8u; + ResW& rw = resident(W); + if (!rw.ok) return false; + + size_t code_bytes = (size_t)ng*gs*sizeof(__fp16); + if (!g_code_buf || (size_t)g_code_buf.length < code_bytes) + g_code_buf = owned_shared(code_bytes); + if (!g_code_buf) return false; + ensureEncoder(); + bool simdT = (gs==128u && ctx().psoTsimd); + [g_enc setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; + setBufAt(lhs, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:1]; + [g_enc setBuffer:rw.lsign offset:rw.ls_off atIndex:2]; [g_enc setBuffer:rw.rsign offset:rw.rs_off atIndex:3]; + [g_enc setBuffer:rw.perm offset:rw.pm_off atIndex:4]; [g_enc setBuffer:g_code_buf offset:0 atIndex:5]; + [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; + if (lowbit && bits == 2u && gs == 128u && ctx().psoActQ && ctx().psoG2i8 && rw.cb_i8) { + id actq = recycled((size_t)ng*gs); + id ascl = recycled((size_t)ng*sizeof(float)); + [g_enc setComputePipelineState:ctx().psoActQ]; + [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; + [g_enc setBuffer:actq offset:0 atIndex:1]; [g_enc setBuffer:ascl offset:0 atIndex:2]; + [g_enc setBytes:&gs length:4 atIndex:3]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + [g_enc setComputePipelineState:ctx().psoG2i8]; + [g_enc setBuffer:actq offset:0 atIndex:0]; [g_enc setBuffer:ascl offset:0 atIndex:1]; + [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:2]; [g_enc setBuffer:rw.cb_i8 offset:0 atIndex:3]; + [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:4]; + setBufAt(out, (size_t)N*2, 5); + [g_enc setBytes:&ng length:4 atIndex:6]; [g_enc setBytes:&N length:4 atIndex:7]; + [g_enc setBytes:&rw.cb_scale length:4 atIndex:8]; + [g_enc dispatchThreadgroups:MTLSizeMake((N+7u)/8u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; + } + bool umr = ctx().psoGmr && (N % g_mr_rows == 0u); + [g_enc setComputePipelineState:(lowbit?ctx().psoGmrLow:(umr?ctx().psoGmr:ctx().psoG))]; + [g_enc setBuffer:g_code_buf offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:1]; + [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:2]; [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:3]; + setBufAt(out, (size_t)N*2, 4); + [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; + [g_enc setBytes:&pgbw length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; + if (lowbit) { [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc setBytes:&rw.il length:4 atIndex:10]; } + else [g_enc setBytes:&rw.il length:4 atIndex:9]; + uint32_t ROWS=8; + uint32_t grid = (lowbit || umr) ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; + [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + return true; +} + +bool cactus_metal_encode_transform_batch(const void* x, const CactusQuantMatrix* const* Ws, int B, void* const* codes) { + if (!ctx().ok || B < 1 || B > 3 || !ctx().psoTbatch) return false; + const uint32_t gs=Ws[0]->group_size, K=Ws[0]->K, ng=Ws[0]->num_groups; + if (gs != 128u || !quant_fast_eligible(Ws[0])) return false; + ResW& r0 = resident(Ws[0]); + ResW* rw[3] = { &r0, &r0, &r0 }; + for (int b=1;bok) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoTbatch]; + setBufAt(x, (size_t)K*2, 0); + [g_enc setBuffer:r0.recip offset:r0.rc_off atIndex:1]; + for (int b=0;b<3;b++) { + ResW* R = rw[blsign offset:R->ls_off atIndex:2+b]; + [g_enc setBuffer:R->rsign offset:R->rs_off atIndex:5+b]; + [g_enc setBuffer:R->perm offset:R->pm_off atIndex:8+b]; + setBufAt(codes[bgroup_size, K=W->K, N=W->N, ng=W->num_groups, pgb=(gs*4u+7u)/8u; + if (!quant_fast_eligible(W)) return false; + ResW& rw = resident(W); + if (!rw.ok) return false; + ensureEncoder(); + bool umr = ctx().psoGmr && (N % g_mr_rows == 0u); + [g_enc setComputePipelineState:(umr?ctx().psoGmr:ctx().psoG)]; + setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:1]; + [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:2]; [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:3]; + setBufAt(out, (size_t)N*2, 4); + [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; + [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; + [g_enc setBytes:&rw.il length:4 atIndex:9]; + uint32_t ROWS=8; + uint32_t grid = umr ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; + [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + return true; +} + +bool cactus_metal_transform_gemv_fits(uint32_t K) { + return ctx().ok && tg_mem_ok((size_t)K*2 + 8*128*sizeof(float) + 64); +} + +bool cactus_metal_encode_transform_gemv(void* out, const void* x, const CactusQuantMatrix* W, const void* osw) { + if (!ctx().ok || !ctx().psoMega) return false; + const uint32_t gs=W->group_size, K=W->K, ng=W->num_groups, N=W->N; + if (gs != 128u || !quant_fast_eligible(W) || (N % 16u) != 0u) return false; + if (!cactus_metal_transform_gemv_fits(K)) return false; + ResW& rw = resident(W); + if (!rw.ok) return false; + ensureEncoder(); + struct { uint32_t K, ng, oswi; } U = { K, ng, (uint32_t)(osw ? 1 : 0) }; + [g_enc setComputePipelineState:ctx().psoMega]; + setBufAt(x, (size_t)K*2, 0); + [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:1]; + [g_enc setBuffer:rw.lsign offset:rw.ls_off atIndex:2]; [g_enc setBuffer:rw.rsign offset:rw.rs_off atIndex:3]; + [g_enc setBuffer:rw.perm offset:rw.pm_off atIndex:4]; [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:5]; + [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:6]; [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:7]; + setBufAt(out, (size_t)N*2, 8); + if (osw) setBufAt(osw, (size_t)N*2, 9); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:9]; + [g_enc setBytes:&U length:sizeof(U) atIndex:10]; + [g_enc setThreadgroupMemoryLength:(size_t)K*2 atIndex:0]; + [g_enc setThreadgroupMemoryLength:8*128*sizeof(float) atIndex:1]; + [g_enc dispatchThreadgroups:MTLSizeMake(N/16u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} + +bool cactus_metal_encode_swiglu_transform(void* code, const void* gate, const void* up, + const CactusQuantMatrix* W, float scale) { + if (!ctx().ok || !ctx().psoSwiT || W->group_size != 128u || !quant_fast_eligible(W)) return false; + const uint32_t K=W->K, ng=W->num_groups; + ResW& rw = resident(W); + if (!rw.ok) return false; + ensureEncoder(); + float s = scale; + [g_enc setComputePipelineState:ctx().psoSwiT]; + setBufAt(gate, (size_t)K*2, 0); setBufAt(up, (size_t)K*2, 1); + [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:2]; + [g_enc setBuffer:rw.lsign offset:rw.ls_off atIndex:3]; [g_enc setBuffer:rw.rsign offset:rw.rs_off atIndex:4]; + [g_enc setBuffer:rw.perm offset:rw.pm_off atIndex:5]; + setBufAt(code, (size_t)K*2, 6); + [g_enc setBytes:&s length:4 atIndex:7]; + [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + return true; +} + +bool cactus_metal_encode_gemv_cat(void* const* outs, const void* const* codes, + const CactusQuantMatrix* const* Ws, int B) { + if (!ctx().ok || !ctx().psoCat || B < 1 || B > 3) return false; + const uint32_t gs=Ws[0]->group_size, K=Ws[0]->K, ng=Ws[0]->num_groups; + if (gs != 128u) return false; + uint32_t Ns[3]={0,0,0}; + for (int i=0;iK != K || (Ws[i]->N % 16u) != 0u) return false; + Ns[i]=Ws[i]->N; + } + ResW* rw[3]; rw[0]=&resident(Ws[0]); rw[1]=rw[0]; rw[2]=rw[0]; + for (int i=1;iok) return false; + ensureEncoder(); + struct { uint32_t ng, N0, N1, N2; } U = { ng, Ns[0], Ns[1], Ns[2] }; + [g_enc setComputePipelineState:ctx().psoCat]; + for (int i=0;i<3;i++){ + int j = ipacked offset:rw[j]->packed_off atIndex:3+i]; + [g_enc setBuffer:rw[j]->codebook offset:rw[j]->cb_off atIndex:6+i]; + [g_enc setBuffer:rw[j]->norms offset:rw[j]->norms_off atIndex:9+i]; + setBufAt(outs[j], (size_t)Ns[j]*2, 12+i); + } + [g_enc setBytes:&U length:sizeof(U) atIndex:15]; + uint32_t total = Ns[0]+Ns[1]+Ns[2]; + [g_enc dispatchThreadgroups:MTLSizeMake(total/16u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} + +bool cactus_metal_prewarm_quant(const CactusQuantMatrix* W) { + if (!ctx().ok) return false; + if (!quant_fast_eligible(W) && !quant_lowbit_eligible(W)) return false; + return resident(W).ok; +} + +namespace { +bool moe_component_offsets(const CactusQuantMatrix& W, uintptr_t& lo, uintptr_t& hi, size_t offs[7]) { + const uintptr_t pg = 16384; + struct Span { const void* p; size_t len; }; + Span spans[7] = { + { W.packed_indices, (size_t)W.N*W.num_groups*64u }, + { W.norms, (size_t)W.N*W.num_groups*2 }, + { W.input_scale_recip, (size_t)W.K*2 }, + { W.left_signs, 128 }, + { W.right_signs, 128 }, + { W.permutation, 512 }, + { W.codebook, 32 }, + }; + uintptr_t mn = UINTPTR_MAX, mx = 0; + for (const auto& s : spans) { + if (!s.p) return false; + uintptr_t a = (uintptr_t)s.p; + if (a < mn) mn = a; + if (a + s.len > mx) mx = a + s.len; + } + lo = mn & ~(pg - 1); + hi = (mx + pg - 1) & ~(pg - 1); + for (int i = 0; i < 7; i++) offs[i] = (uintptr_t)spans[i].p - lo; + return true; +} + +bool moe_build_set4_zerocopy(MoESet4& S, const CactusQuantMatrix* Ws, uint32_t count) { + if (count == 0) return false; + uintptr_t lo0, hi0; + size_t offs[7]; + if (!moe_component_offsets(Ws[0], lo0, hi0, offs)) return false; + const size_t slot = hi0 - lo0; + if (slot > 0xFFFFFFFFull) return false; + if ((offs[0] & 7u) || (offs[1] & 1u) || (offs[2] & 1u) || (offs[6] & 1u) || (offs[5] & 3u)) return false; + for (uint32_t e = 1; e < count; ++e) { + uintptr_t lo, hi; + size_t o[7]; + if (!moe_component_offsets(Ws[e], lo, hi, o)) return false; + if (hi - lo != slot) return false; + for (int i = 0; i < 7; i++) if (o[i] != offs[i]) return false; + } + const size_t total = slot * count; + void* arena = mmap(nullptr, total, PROT_NONE, MAP_ANON | MAP_PRIVATE, -1, 0); + if (arena == MAP_FAILED) return false; + for (uint32_t e = 0; e < count; ++e) { + uintptr_t lo, hi; + size_t o[7]; + moe_component_offsets(Ws[e], lo, hi, o); + vm_address_t dst = (vm_address_t)((uintptr_t)arena + (size_t)e*slot); + vm_prot_t curp = 0, maxp = 0; + kern_return_t kr = vm_remap(mach_task_self(), &dst, slot, 0, + VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, mach_task_self(), + (vm_address_t)lo, FALSE, &curp, &maxp, VM_INHERIT_NONE); + if (kr != KERN_SUCCESS || dst != (vm_address_t)((uintptr_t)arena + (size_t)e*slot) + || !(curp & VM_PROT_READ)) { + munmap(arena, total); + return false; + } + } + id ab = [ctx().dev newBufferWithBytesNoCopy:arena length:total + options:MTLResourceStorageModeShared deallocator:nil]; + if (!ab) { munmap(arena, total); return false; } + CactusThreading::parallel_for(count, CactusThreading::ParallelConfig(1, 1), + [&](size_t start, size_t end) { + volatile uint64_t acc = 0; + for (size_t e = start; e < end; ++e) + for (size_t o = 0; o < slot; o += 16384) + acc += *((const volatile uint8_t*)arena + e*slot + o); + (void)acc; + }); + S.K=Ws[0].K; S.N=Ws[0].N; + S.pk=S.nm=S.rc=S.ls=S.rs=S.pm=S.cb=ab; + S.pk_off=offs[0]; S.nm_off=offs[1]; S.rc_off=offs[2]; S.ls_off=offs[3]; + S.rs_off=offs[4]; S.pm_off=offs[5]; S.cb_off=offs[6]; + S.stride = (uint32_t)slot; + return true; +} + +bool moe_build_set4(MoESet4& S, const CactusQuantMatrix* Ws, uint32_t count) { + const uint32_t K=Ws[0].K, N=Ws[0].N, gs=Ws[0].group_size, ng=Ws[0].num_groups; + if (gs != 128u || Ws[0].bits != 4u || (N % 4u) != 0u || (K % 128u) != 0u) return false; + for (uint32_t e = 0; e < count; ++e) { + const CactusQuantMatrix* W = &Ws[e]; + if (!quant_fast_eligible(W) || W->K != K || W->N != N || W->num_groups != ng) return false; + } + return moe_build_set4_zerocopy(S, Ws, count); +} +} + +bool cactus_metal_moe_cq4_ready(const CactusQuantMatrix* w1_0) { + std::lock_guard lk(g_resident_mu); + auto it = g_moe_cat.find(resident_key(w1_0)); + return it != g_moe_cat.end() && it->second.ok; +} + +bool cactus_metal_moe_cq4_build(const CactusQuantMatrix* w1s, const CactusQuantMatrix* w3s, + const CactusQuantMatrix* w2s, uint32_t num_experts) { + if (!ctx().ok || !ctx().psoMoeT || !ctx().psoMoeT2 || !ctx().psoMoeUp || !ctx().psoMoeDownAcc) + return false; + const uint64_t key = resident_key(&w1s[0]); + { + std::lock_guard lk(g_resident_mu); + auto it = g_moe_cat.find(key); + if (it != g_moe_cat.end()) return it->second.ok; + } + MoECat4 cat; + cat.ok = moe_build_set4(cat.w1, w1s, num_experts) + && moe_build_set4(cat.w3, w3s, num_experts) + && moe_build_set4(cat.w2, w2s, num_experts) + && cat.w1.K == cat.w3.K && cat.w1.N == cat.w3.N && cat.w2.K >= cat.w1.N; + if (!cat.ok) cat = MoECat4{}; + std::lock_guard lk(g_resident_mu); + auto ins = g_moe_cat.emplace(key, cat); + return ins.first->second.ok; +} + +bool cactus_metal_encode_moe_gated_cq4(void* out, const void* hidden, const void* probs, + const void* topk, const CactusQuantMatrix* w1_0, + uint32_t num_experts, uint32_t top_k, uint32_t tokens, + uint32_t act, uint32_t normalize, float eps, float scaling) { + if (!ctx().ok || !ctx().psoMoeT || top_k == 0 || top_k > 16 || tokens == 0) return false; + g_resident_mu.lock(); + auto it = g_moe_cat.find(resident_key(w1_0)); + bool hit = it != g_moe_cat.end() && it->second.ok; + g_resident_mu.unlock(); + if (!hit) return false; + MoECat4& C = it->second; + const uint32_t K1=C.w1.K, N1=C.w1.N, K2=C.w2.K, N2=C.w2.N; + const size_t slots = (size_t)tokens*top_k; + ensureEncoder(); + + id code1 = recycled(slots*K1*2); + id code3 = recycled(slots*K1*2); + id gu = recycled(slots*N1*2); + id code2 = recycled(slots*K2*2); + + { + [g_enc setComputePipelineState:ctx().psoMoeT2]; + setBufAt(hidden, (size_t)tokens*K1*2, 0); + setBufAt(topk, slots*sizeof(float), 1); + [g_enc setBuffer:C.w1.rc offset:C.w1.rc_off atIndex:2]; + [g_enc setBuffer:C.w1.ls offset:C.w1.ls_off atIndex:3]; + [g_enc setBuffer:C.w1.rs offset:C.w1.rs_off atIndex:4]; + [g_enc setBuffer:C.w1.pm offset:C.w1.pm_off atIndex:5]; + [g_enc setBuffer:C.w3.rc offset:C.w3.rc_off atIndex:6]; + [g_enc setBuffer:C.w3.ls offset:C.w3.ls_off atIndex:7]; + [g_enc setBuffer:C.w3.rs offset:C.w3.rs_off atIndex:8]; + [g_enc setBuffer:C.w3.pm offset:C.w3.pm_off atIndex:9]; + [g_enc setBuffer:code1 offset:0 atIndex:10]; + [g_enc setBuffer:code3 offset:0 atIndex:11]; + uint32_t Kc=K1, tk=top_k; + [g_enc setBytes:&Kc length:4 atIndex:12]; [g_enc setBytes:&tk length:4 atIndex:13]; + [g_enc setBytes:&C.w1.stride length:4 atIndex:14]; + [g_enc setBytes:&C.w3.stride length:4 atIndex:15]; + [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(K1/128u, 2u*top_k, tokens) + threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + } + { + [g_enc setComputePipelineState:ctx().psoMoeUp]; + [g_enc setBuffer:code1 offset:0 atIndex:0]; [g_enc setBuffer:code3 offset:0 atIndex:1]; + setBufAt(topk, slots*sizeof(float), 2); + [g_enc setBuffer:C.w1.pk offset:C.w1.pk_off atIndex:3]; [g_enc setBuffer:C.w1.nm offset:C.w1.nm_off atIndex:4]; [g_enc setBuffer:C.w1.cb offset:C.w1.cb_off atIndex:5]; + [g_enc setBuffer:C.w3.pk offset:C.w3.pk_off atIndex:6]; [g_enc setBuffer:C.w3.nm offset:C.w3.nm_off atIndex:7]; [g_enc setBuffer:C.w3.cb offset:C.w3.cb_off atIndex:8]; + [g_enc setBuffer:gu offset:0 atIndex:9]; + uint32_t Kc=K1, Nc=N1, a=act, tk=top_k; + [g_enc setBytes:&Kc length:4 atIndex:10]; [g_enc setBytes:&Nc length:4 atIndex:11]; + [g_enc setBytes:&a length:4 atIndex:12]; [g_enc setBytes:&tk length:4 atIndex:13]; + [g_enc setBytes:&C.w1.stride length:4 atIndex:14]; + [g_enc setBytes:&C.w3.stride length:4 atIndex:15]; + [g_enc dispatchThreadgroups:MTLSizeMake((N1+31u)/32u, top_k, tokens) + threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } + { + [g_enc setComputePipelineState:ctx().psoMoeT]; + [g_enc setBuffer:gu offset:0 atIndex:0]; + setBufAt(topk, slots*sizeof(float), 1); + [g_enc setBuffer:C.w2.rc offset:C.w2.rc_off atIndex:2]; + [g_enc setBuffer:C.w2.ls offset:C.w2.ls_off atIndex:3]; + [g_enc setBuffer:C.w2.rs offset:C.w2.rs_off atIndex:4]; + [g_enc setBuffer:C.w2.pm offset:C.w2.pm_off atIndex:5]; + [g_enc setBuffer:code2 offset:0 atIndex:6]; + uint32_t Kc=K2, kv=N1, xz=top_k*N1, xy=N1, tk=top_k; + [g_enc setBytes:&Kc length:4 atIndex:7]; [g_enc setBytes:&kv length:4 atIndex:8]; + [g_enc setBytes:&xz length:4 atIndex:9]; [g_enc setBytes:&xy length:4 atIndex:10]; + [g_enc setBytes:&tk length:4 atIndex:11]; + [g_enc setBytes:&C.w2.stride length:4 atIndex:12]; + [g_enc setThreadgroupMemoryLength:128*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(K2/128u, top_k, tokens) + threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + } + { + [g_enc setComputePipelineState:ctx().psoMoeDownAcc]; + [g_enc setBuffer:code2 offset:0 atIndex:0]; + setBufAt(topk, slots*sizeof(float), 1); + setBufAt(probs, (size_t)tokens*num_experts*2, 2); + [g_enc setBuffer:C.w2.pk offset:C.w2.pk_off atIndex:3]; [g_enc setBuffer:C.w2.nm offset:C.w2.nm_off atIndex:4]; [g_enc setBuffer:C.w2.cb offset:C.w2.cb_off atIndex:5]; + setBufAt(out, (size_t)tokens*N2*2, 6); + uint32_t Kc=K2, Nc=N2, tk=top_k, nz=normalize, Ec=num_experts; + [g_enc setBytes:&Kc length:4 atIndex:7]; [g_enc setBytes:&Nc length:4 atIndex:8]; + [g_enc setBytes:&tk length:4 atIndex:9]; [g_enc setBytes:&nz length:4 atIndex:10]; + [g_enc setBytes:&eps length:4 atIndex:11]; [g_enc setBytes:&scaling length:4 atIndex:12]; + [g_enc setBytes:&Ec length:4 atIndex:13]; + [g_enc setBytes:&C.w2.stride length:4 atIndex:14]; + [g_enc setThreadgroupMemoryLength:top_k*16*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((N2+15u)/16u, 1, tokens) + threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } + return true; +} + +bool cactus_metal_encode_deltanet_decode(void* out, const void* q, const void* k, const void* v, + const void* g, const void* b, const void* s, + uint32_t B, uint32_t Hq, uint32_t Hv, + uint32_t K, uint32_t V, float scale) { + if (!ctx().ok || !ctx().psoDeltanet) return false; + if (B == 0 || Hq == 0 || Hv == 0 || K == 0 || V == 0 || V > 1024 || (Hv % Hq) != 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoDeltanet]; + setBufAt(q, (size_t)B*Hq*K*2, 0); + setBufAt(k, (size_t)B*Hq*K*2, 1); + setBufAt(v, (size_t)B*Hv*V*2, 2); + setBufAt(g, (size_t)B*Hv*2, 3); + setBufAt(b, (size_t)B*Hv*2, 4); + setBufAt(s, (size_t)B*K*Hv*V*2, 5); + setBufAt(out, (size_t)B*(1+K)*Hv*V*2, 6); + [g_enc setBytes:&Hq length:4 atIndex:7]; [g_enc setBytes:&Hv length:4 atIndex:8]; + [g_enc setBytes:&K length:4 atIndex:9]; [g_enc setBytes:&V length:4 atIndex:10]; + [g_enc setBytes:&scale length:4 atIndex:11]; + [g_enc setThreadgroupMemoryLength:(size_t)K*4 atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(Hv, B, 1) threadsPerThreadgroup:MTLSizeMake(V, 1, 1)]; + return true; +} + +bool cactus_metal_encode_deltanet_prefill(void* out, const void* q, const void* k, const void* v, + const void* g, const void* b, const void* s, + uint32_t B, uint32_t T, uint32_t Hq, uint32_t Hv, + uint32_t K, uint32_t V, float scale) { + if (!ctx().ok || !ctx().psoDeltanetPre) return false; + if (B == 0 || T == 0 || Hq == 0 || Hv == 0 || K == 0 || V == 0 || V > 1024 || (Hv % Hq) != 0) return false; + ensureEncoder(); + id scratch = recycled((size_t)B*Hv*K*V*sizeof(float)); + [g_enc setComputePipelineState:ctx().psoDeltanetPre]; + setBufAt(q, (size_t)B*T*Hq*K*2, 0); + setBufAt(k, (size_t)B*T*Hq*K*2, 1); + setBufAt(v, (size_t)B*T*Hv*V*2, 2); + setBufAt(g, (size_t)B*T*Hv*2, 3); + setBufAt(b, (size_t)B*T*Hv*2, 4); + setBufAt(s, (size_t)B*K*Hv*V*2, 5); + setBufAt(out, (size_t)B*(T+K)*Hv*V*2, 6); + [g_enc setBuffer:scratch offset:0 atIndex:7]; + [g_enc setBytes:&T length:4 atIndex:8]; [g_enc setBytes:&Hq length:4 atIndex:9]; + [g_enc setBytes:&Hv length:4 atIndex:10]; [g_enc setBytes:&K length:4 atIndex:11]; + [g_enc setBytes:&V length:4 atIndex:12]; [g_enc setBytes:&scale length:4 atIndex:13]; + [g_enc setThreadgroupMemoryLength:(size_t)K*4 atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(Hv, B, 1) threadsPerThreadgroup:MTLSizeMake(V, 1, 1)]; + return true; +} + +bool cactus_metal_encode_rms2_add_clip(void* out, const void* a, const void* wa, + const void* b, const void* wb, size_t dim, + float eps_a, float eps_b) { + if (!ctx().ok || !ctx().psoRms2AddClip || dim == 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoRms2AddClip]; + setBufAt(a, dim*2, 0); setBufAt(wa, dim*2, 1); + setBufAt(b, dim*2, 2); setBufAt(wb, dim*2, 3); + setBufAt(out, dim*2, 4); + uint32_t d=(uint32_t)dim; float ea=eps_a, eb=eps_b; + [g_enc setBytes:&d length:4 atIndex:5]; [g_enc setBytes:&ea length:4 atIndex:6]; + [g_enc setBytes:&eb length:4 atIndex:7]; + [g_enc setThreadgroupMemoryLength:512*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(1,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} + +bool cactus_metal_encode_rms_norm_scale(void* out, const void* in, const void* weight, + size_t rows, size_t dim, float eps, float oscale) { + if (!ctx().ok || !ctx().psoRmsScale || rows == 0 || dim == 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoRmsScale]; + setBufAt(in, rows*dim*2, 0); setBufAt(weight, dim*2, 1); setBufAt(out, rows*dim*2, 2); + uint32_t d=(uint32_t)dim; float e=eps, sc=oscale; + [g_enc setBytes:&d length:4 atIndex:3]; [g_enc setBytes:&e length:4 atIndex:4]; + [g_enc setBytes:&sc length:4 atIndex:5]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} + +bool cactus_metal_encode_softmax_topk(void* probs, void* topk, const void* in, + size_t rows, size_t cols, size_t k, float scale) { + if (!ctx().ok || !ctx().psoSoftmaxTopk || k == 0 || k > 16 || rows == 0 || cols == 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoSoftmaxTopk]; + setBufAt(in, rows*cols*2, 0); + setBufAt(probs, rows*cols*2, 1); + setBufAt(topk, rows*k*2*sizeof(float), 2); + uint32_t E=(uint32_t)cols, kc=(uint32_t)k, B=(uint32_t)rows; float sc=scale; + [g_enc setBytes:&E length:4 atIndex:3]; + [g_enc setBytes:&kc length:4 atIndex:4]; + [g_enc setBytes:&B length:4 atIndex:5]; + [g_enc setBytes:&sc length:4 atIndex:6]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + return true; +} + +bool cactus_metal_encode_rope_pair(void* out, const void* x, const void* c, const void* s, + uint32_t H, uint32_t D) { + if (!ctx().ok || !ctx().psoRopePair || H == 0 || D == 0 || (D % 2) != 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoRopePair]; + setBufAt(x, (size_t)H*D*2, 0); + setBufAt(c, (size_t)D*2, 1); + setBufAt(s, (size_t)D*2, 2); + setBufAt(out, (size_t)H*D*2, 3); + [g_enc setBytes:&H length:4 atIndex:4]; + [g_enc setBytes:&D length:4 atIndex:5]; + [g_enc dispatchThreads:MTLSizeMake(D, H, 1) threadsPerThreadgroup:MTLSizeMake(64, 4, 1)]; + return true; +} + +bool cactus_metal_encode_rope_pair_rms(void* out, const void* x, const void* w, + const void* c, const void* s, + uint32_t H, uint32_t D, float eps) { + if (!ctx().ok || !ctx().psoRopePairRms || H == 0 || D == 0 || (D % 2) != 0 || D > 1024) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoRopePairRms]; + setBufAt(x, (size_t)H*D*2, 0); + setBufAt(w, (size_t)D*2, 1); + setBufAt(c, (size_t)D*2, 2); + setBufAt(s, (size_t)D*2, 3); + setBufAt(out, (size_t)H*D*2, 4); + float e = eps; + [g_enc setBytes:&D length:4 atIndex:5]; + [g_enc setBytes:&e length:4 atIndex:6]; + uint32_t T = D < 256 ? D : 256; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:(size_t)D*2 atIndex:1]; + [g_enc dispatchThreadgroups:MTLSizeMake(H,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + return true; +} + +bool cactus_metal_encode_topk_rows(void* out, const void* in, size_t rows, size_t cols, size_t k) { + if (!ctx().ok || !ctx().psoTopkRows || k == 0 || k > 16 || rows == 0 || cols == 0) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoTopkRows]; + setBufAt(in, rows*cols*2, 0); + setBufAt(out, rows*k*2*sizeof(float), 1); + uint32_t F=(uint32_t)cols, kc=(uint32_t)k, B=(uint32_t)rows; + [g_enc setBytes:&F length:4 atIndex:2]; + [g_enc setBytes:&kc length:4 atIndex:3]; + [g_enc setBytes:&B length:4 atIndex:4]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + return true; +} + +bool cactus_metal_encode_quant_matmul_m(void* out, const void* lhs, const CactusQuantMatrix* W, uint32_t M) { + if (!ctx().ok || !W) return false; + if (M == 1 && (W->flags & CACTUS_QUANT_FLAG_ORTHOGONAL)) + return cactus_metal_encode_quant_matmul(out, lhs, W); + + const uint32_t gs=W->group_size, K=W->K, N=W->N, ng=W->num_groups; + const uint32_t bits=W->bits, pgb=(gs*bits+7u)/8u; + const bool fast = ctx().ok && quant_fast_eligible(W); + if (!fast || !ctx().psoGmma) return false; + ResW& rw = resident(W); + if (!rw.ok) return false; + size_t code_bytes = (size_t)M*ng*gs*sizeof(__fp16); + if (!g_code_buf_m || (size_t)g_code_buf_m.length < code_bytes) + g_code_buf_m = owned_shared(code_bytes); + if (!g_code_buf_m) return false; + ensureEncoder(); + + [g_enc setComputePipelineState:ctx().psoTm]; + setBufAt(lhs, (size_t)M*K*2, 0); [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:1]; + [g_enc setBuffer:rw.lsign offset:rw.ls_off atIndex:2]; [g_enc setBuffer:rw.rsign offset:rw.rs_off atIndex:3]; + [g_enc setBuffer:rw.perm offset:rw.pm_off atIndex:4]; [g_enc setBuffer:g_code_buf_m offset:0 atIndex:5]; + [g_enc setBytes:&gs length:4 atIndex:6]; [g_enc setBytes:&K length:4 atIndex:7]; + [g_enc setThreadgroupMemoryLength:gs*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs>1024u?1024u:gs,1,1)]; + + [g_enc setComputePipelineState:ctx().psoGmma]; + [g_enc setBuffer:g_code_buf_m offset:0 atIndex:0]; [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:1]; + [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:2]; [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:3]; + setBufAt(out, (size_t)M*N*2, 4); + [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; + [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; [g_enc setBytes:&M length:4 atIndex:9]; + [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + return true; +} + +bool cactus_metal_encode_quant_matmul_ortho(void* out, const void* act, void* code, + const CactusQuantMatrix* W) { + if (!ctx().ok || !W->rotation) return false; + const uint32_t K=W->K, N=W->N, ng=W->num_groups, gs=W->group_size, bits=W->bits; + const uint32_t pgb=(gs*bits+7u)/8u; + const bool lowbit = (bits == 2 || bits == 3); + if (bits != 4 && !lowbit) return false; + if (lowbit && (!ctx().psoGmrLow || (N % 16) != 0)) return false; + if (ng != 1 || gs != K || (N % 4) != 0) return false; + ResW& rw = resident(W); + if (!rw.ok || !rw.rotation || !rw.packed) return false; + ensureEncoder(); + + [g_enc setComputePipelineState:ctx().psoRotW]; + setBufAt(act, (size_t)K*2, 0); [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:1]; + [g_enc setBuffer:rw.rotation offset:rw.rot_off atIndex:2]; setBufAt(code, (size_t)K*2, 3); + [g_enc setBytes:&K length:4 atIndex:4]; + [g_enc setThreadgroupMemoryLength:256*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((K+31)/32,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + + bool umr = ctx().psoGmr && (N % g_mr_rows == 0u); + [g_enc setComputePipelineState:(lowbit?ctx().psoGmrLow:(umr?ctx().psoGmr:ctx().psoG))]; + setBufAt(code, (size_t)K*2, 0); [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:1]; + [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:2]; [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:3]; + setBufAt(out, (size_t)N*2, 4); + [g_enc setBytes:&gs length:4 atIndex:5]; [g_enc setBytes:&ng length:4 atIndex:6]; + [g_enc setBytes:&pgb length:4 atIndex:7]; [g_enc setBytes:&N length:4 atIndex:8]; + if (lowbit) { [g_enc setBytes:&bits length:4 atIndex:9]; [g_enc setBytes:&rw.il length:4 atIndex:10]; } + else [g_enc setBytes:&rw.il length:4 atIndex:9]; + uint32_t ROWS=8; + uint32_t grid = (lowbit || umr) ? (N+g_mr_rows-1u)/g_mr_rows : (N+ROWS-1u)/ROWS; + [g_enc dispatchThreadgroups:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + return true; +} + +bool cactus_metal_encode_embedding_ortho(void* out, uint32_t row, const CactusQuantMatrix* W, float scale) { + if (!ctx().ok || !W->rotation || (W->bits != 4 && W->bits != 2 && W->bits != 3)) return false; + const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size, bits=W->bits; + if (ng != 1 || gs != K) return false; + if (!tg_mem_ok((size_t)K*sizeof(float))) return false; + if (bits != 4 && !(ctx().psoEmbOW && (K % 8u) == 0u)) return false; + if (!ctx().psoEmbOW || (K % 8u) != 0u) return false; + ResW& rw = resident(W); + if (!rw.ok || !rw.packed || !rw.rotation || !rw.codebook || !rw.norms || !rw.recip) return false; + ensureEncoder(); + { + [g_enc setComputePipelineState:ctx().psoEmbOW]; + [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:0]; [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:1]; + [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:2]; [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:3]; + [g_enc setBuffer:rw.rotation offset:rw.rot_off atIndex:4]; setBufAt(out, (size_t)K*2, 5); + [g_enc setBytes:&K length:4 atIndex:6]; [g_enc setBytes:&row length:4 atIndex:7]; + [g_enc setBytes:&scale length:4 atIndex:8]; [g_enc setBytes:&bits length:4 atIndex:9]; + [g_enc setBytes:&rw.il length:4 atIndex:10]; + [g_enc setThreadgroupMemoryLength:(size_t)K*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(K/8u,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; + } +} + +ResW& resident_emb_meta(const CactusQuantMatrix* W) { + std::lock_guard lk(g_resident_mu); + auto it = g_resident_emb.find(W->packed_indices); + if (it != g_resident_emb.end()) return it->second; + const uint32_t K=W->K, gs=W->group_size, ng=W->num_groups, bits=W->bits; + const uint32_t pgb=(gs*bits+7u)/8u; + ResW r; + bind_component(W->codebook, (size_t)(1u<bits)*sizeof(__fp16), 2, r.codebook, r.cb_off); + bind_component(W->left_signs, gs, 1, r.lsign, r.ls_off); + bind_component(W->right_signs, gs, 1, r.rsign, r.rs_off); + bind_component(W->permutation, (size_t)gs*sizeof(uint32_t), 4, r.perm, r.pm_off); + bind_component(W->input_scale_recip, (size_t)K*sizeof(__fp16), 2, r.recip, r.rc_off); + if (W->N && ng) { + auto wp = wrapHostPtr(W->packed_indices, (size_t)W->N*ng*pgb); + if (wp.first && ((uintptr_t)W->norms & 1u) == 0) { + auto wn = wrapHostPtr(W->norms, (size_t)W->N*ng*sizeof(__fp16)); + if (wn.first) { + r.packed = wp.first; r.packed_off = wp.second; + r.norms = wn.first; r.norms_off = wn.second; + } + } + } + return g_resident_emb.emplace(W->packed_indices, r).first->second; +} + +bool cactus_metal_encode_embedding_hadamard(void* out, uint32_t row, const CactusQuantMatrix* W) { + if (!ctx().ok || (W->bits != 4 && W->bits != 2 && W->bits != 3) || + (W->flags & (CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW))) return false; + const uint32_t K=W->K, gs=W->group_size, ng=W->num_groups, bits=W->bits, pgb=(gs*bits+7u)/8u; + if (gs > 256 || (gs & (gs-1)) != 0 || !W->packed_indices || !W->norms || !W->codebook + || !W->left_signs || !W->right_signs || !W->permutation || !W->input_scale_recip) return false; + ResW& rm = resident_emb_meta(W); + if (!rm.codebook || !rm.recip || !rm.lsign || !rm.rsign || !rm.perm) return false; + ensureEncoder(); + size_t row_bytes = (size_t)ng*pgb; + if (!rm.packed || !rm.norms) return false; + [g_enc setComputePipelineState:ctx().psoEmbH]; + [g_enc setBuffer:rm.packed offset:rm.packed_off + (size_t)row*row_bytes atIndex:0]; + [g_enc setBuffer:rm.norms offset:rm.norms_off + (size_t)row*ng*sizeof(__fp16) atIndex:2]; + [g_enc setBuffer:rm.codebook offset:rm.cb_off atIndex:1]; + [g_enc setBuffer:rm.recip offset:rm.rc_off atIndex:3]; + [g_enc setBuffer:rm.lsign offset:rm.ls_off atIndex:4]; [g_enc setBuffer:rm.rsign offset:rm.rs_off atIndex:5]; + [g_enc setBuffer:rm.perm offset:rm.pm_off atIndex:6]; setBufAt(out, (size_t)K*2, 7); + [g_enc setBytes:&gs length:4 atIndex:8]; [g_enc setBytes:&bits length:4 atIndex:9]; + [g_enc setThreadgroupMemoryLength:(size_t)gs*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(gs,1,1)]; + return true; +} + +static id registerReadonly(const void* p, size_t bytes) { + if (!p) return nil; + auto it = g_readonly.find(p); + if (it != g_readonly.end()) return it->second; + uintptr_t a = (uintptr_t)p, base = a & ~(uintptr_t)16383u; + size_t wraplen = (((a - base) + bytes + 16383u) & ~(size_t)16383u); + id b = [ctx().dev newBufferWithBytesNoCopy:(void*)base length:wraplen + options:MTLResourceStorageModeShared deallocator:nil]; + if (b) g_readonly[p] = b; + return b; +} + +bool cactus_metal_encode_embedding_ortho_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M) { + if (!ctx().ok || !ctx().psoEmbOm || !W->rotation || (W->bits != 4 && W->bits != 2 && W->bits != 3)) return false; + const uint32_t K=W->K, ng=W->num_groups, gs=W->group_size, bits=W->bits; + if (ng != 1 || gs != K) return false; + ResW& rw = resident(W); + if (!rw.packed || !rw.rotation || !rw.codebook || !rw.norms || !rw.recip) return false; + ensureEncoder(); + id rb = recycled((size_t)M*sizeof(uint32_t)); + std::memcpy([rb contents], rows, (size_t)M*sizeof(uint32_t)); + [g_enc setComputePipelineState:ctx().psoEmbOm]; + [g_enc setBuffer:rw.packed offset:rw.packed_off atIndex:0]; [g_enc setBuffer:rw.codebook offset:rw.cb_off atIndex:1]; + [g_enc setBuffer:rw.norms offset:rw.norms_off atIndex:2]; [g_enc setBuffer:rw.recip offset:rw.rc_off atIndex:3]; + [g_enc setBuffer:rw.rotation offset:rw.rot_off atIndex:4]; [g_enc setBuffer:rb offset:0 atIndex:5]; + setBufAt(out, (size_t)M*K*2, 6); [g_enc setBytes:&K length:4 atIndex:7]; [g_enc setBytes:&M length:4 atIndex:8]; + [g_enc setBytes:&bits length:4 atIndex:9]; + [g_enc setBytes:&rw.il length:4 atIndex:10]; + [g_enc dispatchThreadgroups:MTLSizeMake(((K+15)/16)*((M+15)/16),1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} + +bool cactus_metal_encode_embedding_hadamard_m(void* out, const CactusQuantMatrix* W, const uint32_t* rows, uint32_t M) { + if (!ctx().ok || !ctx().psoEmbHm || (W->bits != 4 && W->bits != 2 && W->bits != 3) || + (W->flags & (CACTUS_QUANT_FLAG_ORTHOGONAL | CACTUS_QUANT_FLAG_INTERLEAVED_4ROW))) return false; + const uint32_t K=W->K, gs=W->group_size, ng=W->num_groups, bits=W->bits, pgb=(gs*bits+7u)/8u; + if (gs > 256 || (gs & (gs-1)) != 0 || !W->packed_indices || !W->norms || !W->codebook + || !W->left_signs || !W->right_signs || !W->permutation || !W->input_scale_recip) return false; + ResW& rm = resident_emb_meta(W); + if (!rm.codebook || !rm.recip || !rm.lsign || !rm.rsign || !rm.perm) return false; + ensureEncoder(); + size_t rowb = (size_t)ng*pgb; + id pk = recycled((size_t)M*rowb); + id nm = recycled((size_t)M*ng*sizeof(__fp16)); + for (uint32_t m=0; mpacked_indices+(size_t)rows[m]*rowb, rowb); + std::memcpy((__fp16*)[nm contents]+(size_t)m*ng, (const __fp16*)W->norms+(size_t)rows[m]*ng, (size_t)ng*sizeof(__fp16)); + } + [g_enc setComputePipelineState:ctx().psoEmbHm]; + [g_enc setBuffer:pk offset:0 atIndex:0]; [g_enc setBuffer:rm.codebook offset:rm.cb_off atIndex:1]; + [g_enc setBuffer:nm offset:0 atIndex:2]; [g_enc setBuffer:rm.recip offset:rm.rc_off atIndex:3]; + [g_enc setBuffer:rm.lsign offset:rm.ls_off atIndex:4]; [g_enc setBuffer:rm.rsign offset:rm.rs_off atIndex:5]; + [g_enc setBuffer:rm.perm offset:rm.pm_off atIndex:6]; setBufAt(out, (size_t)M*K*2, 7); + [g_enc setBytes:&gs length:4 atIndex:8]; [g_enc setBytes:&ng length:4 atIndex:9]; [g_enc setBytes:&K length:4 atIndex:10]; + [g_enc setBytes:&bits length:4 atIndex:11]; + [g_enc setThreadgroupMemoryLength:(size_t)gs*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((size_t)ng*M,1,1) threadsPerThreadgroup:MTLSizeMake(gs,1,1)]; + return true; +} + + +bool cactus_metal_encode_gather_f16(void* out, const void* table, size_t table_bytes, + const uint32_t* rows, uint32_t M, uint32_t D) { + if (!ctx().ok || !ctx().psoGather || M == 0 || D == 0) return false; + id tb = registerReadonly(table, table_bytes); + if (!tb) return false; + size_t toff = (uintptr_t)table & (uintptr_t)16383u; + ensureEncoder(); + id rb = recycled((size_t)M*sizeof(uint32_t)); + std::memcpy([rb contents], rows, (size_t)M*sizeof(uint32_t)); + uint32_t n = M*D; + [g_enc setComputePipelineState:ctx().psoGather]; + [g_enc setBuffer:tb offset:toff atIndex:0]; [g_enc setBuffer:rb offset:0 atIndex:1]; + setBufAt(out, (size_t)n*2, 2); + [g_enc setBytes:&D length:4 atIndex:3]; [g_enc setBytes:&n length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} + +static void setCacheAt(const void* p, size_t bytes, int idx) { + if (p && bytes) setBufAt(p, bytes, idx); + else [g_enc setBuffer:ctx().dummy offset:0 atIndex:idx]; +} + +bool cactus_metal_encode_attention_i8( + void* out, const void* q, const void* knew, const void* vnew, + const void* kc, const void* vc, const void* ks, const void* vs, + uint32_t num_q_heads, uint32_t num_kv_heads, uint32_t head_dim, uint32_t v_hdim, + uint32_t history_len, uint32_t total_keys, uint32_t kv_start, uint32_t kv_end, + float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes) { + if (!ctx().ok) return false; + if (kv_end <= kv_start || (num_q_heads % num_kv_heads) != 0) return false; + if (head_dim > 512u || v_hdim > 512u) return false; + ensureEncoder(); + auto setInputs = [&]() { + setBufAt(q, (size_t)num_q_heads*head_dim*2, 0); + if (total_keys > history_len && knew && vnew) { + setBufAt(knew, (size_t)num_kv_heads*head_dim*2, 1); + setBufAt(vnew, (size_t)num_kv_heads*v_hdim*2, 2); + } else { + [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + } + setCacheAt(kc, kc_bytes, 3); setCacheAt(vc, vc_bytes, 4); + setCacheAt(ks, ks_bytes, 5); setCacheAt(vs, vs_bytes, 6); + }; + + const uint32_t T = 256, nsg = T / 32u; + const uint32_t R = kv_end - kv_start; + uint32_t nwg = R / 24u; if (nwg < 1u) nwg = 1u; if (nwg > 32u) nwg = 32u; + id partO = ctx().dummy, partML = ctx().dummy; + if (nwg > 1u) { + partO = recycled((size_t)num_q_heads*nwg*v_hdim*sizeof(float)); + partML = recycled((size_t)num_q_heads*nwg*2*sizeof(float)); + } + [g_enc setComputePipelineState:ctx().psoAttn]; + setInputs(); + setBufAt(out, (size_t)num_q_heads*v_hdim*2, 7); + [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; + [g_enc setBytes:&head_dim length:4 atIndex:10]; [g_enc setBytes:&v_hdim length:4 atIndex:11]; + [g_enc setBytes:&history_len length:4 atIndex:12];[g_enc setBytes:&scale length:4 atIndex:13]; + [g_enc setBytes:&kv_start length:4 atIndex:14]; [g_enc setBytes:&kv_end length:4 atIndex:15]; + [g_enc setBuffer:partO offset:0 atIndex:16]; [g_enc setBuffer:partML offset:0 atIndex:17]; + [g_enc setBytes:&nwg length:4 atIndex:18]; + [g_enc setThreadgroupMemoryLength:((size_t)nsg*v_hdim + 2*nsg)*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + if (nwg > 1u) { + [g_enc setComputePipelineState:ctx().psoAttnC]; + [g_enc setBuffer:partO offset:0 atIndex:0]; [g_enc setBuffer:partML offset:0 atIndex:1]; + setBufAt(out, (size_t)num_q_heads*v_hdim*2, 2); + [g_enc setBytes:&v_hdim length:4 atIndex:3]; [g_enc setBytes:&nwg length:4 atIndex:4]; + [g_enc dispatchThreadgroups:MTLSizeMake(num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } + return true; +} + +bool cactus_metal_encode_attention_fused_i8( + void* out, const void* q, const void* kraw, const void* vraw, + void* kc, void* vc, void* ks, void* vs, + const void* qw, const void* kw, const void* vw, const void* cs, const void* sn, + uint32_t nqh, uint32_t hd, uint32_t vhd, + uint32_t kv_start, uint32_t kv_end, uint32_t slot, uint32_t has_new, + float eps, float scale, + size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes) { + if (!ctx().ok || !ctx().psoAttnF) return false; + if (kv_end <= kv_start) return false; + ensureEncoder(); + const uint32_t T = 256, nsg = T / 32u; + const uint32_t R = kv_end - kv_start; + uint32_t nwg = R / 24u; if (nwg < 1u) nwg = 1u; if (nwg > 32u) nwg = 32u; + id partO = ctx().dummy, partML = ctx().dummy; + if (nwg > 1u) { + partO = recycled((size_t)nqh*nwg*vhd*sizeof(float)); + partML = recycled((size_t)nqh*nwg*2*sizeof(float)); + } + struct { uint32_t nqh, nkvh, hd, vhd, hist, kv_start, kv_end, nwg, slot, has_new; float eps, scale; } U = + { nqh, 1u, hd, vhd, 0u, kv_start, kv_end, nwg, slot, has_new, eps, scale }; + [g_enc setComputePipelineState:ctx().psoAttnF]; + setBufAt(q, (size_t)nqh*hd*2, 0); + if (has_new && kraw && vraw) { setBufAt(kraw, (size_t)hd*2, 1); setBufAt(vraw, (size_t)vhd*2, 2); } + else { [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; } + setBufAt(kc, kc_bytes, 3); setBufAt(vc, vc_bytes, 4); + setBufAt(ks, ks_bytes, 5); setBufAt(vs, vs_bytes, 6); + setBufAt(out, (size_t)nqh*vhd*2, 7); + setBufAt(qw, (size_t)hd*2, 8); + if (has_new && kw) setBufAt(kw, (size_t)hd*2, 9); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:9]; + if (has_new && vw) setBufAt(vw, (size_t)vhd*2, 10); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:10]; + setBufAt(cs, (size_t)hd*2, 11); setBufAt(sn, (size_t)hd*2, 12); + [g_enc setBuffer:partO offset:0 atIndex:13]; [g_enc setBuffer:partML offset:0 atIndex:14]; + [g_enc setBytes:&U length:sizeof(U) atIndex:15]; + [g_enc setThreadgroupMemoryLength:((size_t)nsg*vhd + 2*nsg + 256)*sizeof(float) atIndex:0]; + [g_enc setThreadgroupMemoryLength:(size_t)(2*hd + vhd)*2 atIndex:1]; + [g_enc dispatchThreadgroups:MTLSizeMake(nqh*nwg,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + if (nwg > 1u) { + [g_enc setComputePipelineState:ctx().psoAttnC]; + [g_enc setBuffer:partO offset:0 atIndex:0]; [g_enc setBuffer:partML offset:0 atIndex:1]; + setBufAt(out, (size_t)nqh*vhd*2, 2); + [g_enc setBytes:&vhd length:4 atIndex:3]; [g_enc setBytes:&nwg length:4 atIndex:4]; + [g_enc dispatchThreadgroups:MTLSizeMake(nqh,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } + return true; +} + +bool cactus_metal_encode_attention_i8_prefill( + void* out, const void* q, const void* knew, const void* vnew, + const void* kc, const void* vc, const void* ks, const void* vs, + uint32_t num_q_heads, uint32_t num_kv_heads, uint32_t head_dim, uint32_t v_hdim, + uint32_t history_len, uint32_t new_len, uint32_t q_pos0, uint32_t window, uint32_t is_causal, uint32_t M, + float scale, size_t kc_bytes, size_t vc_bytes, size_t ks_bytes, size_t vs_bytes, + uint32_t sink, uint32_t ring) { + if (!ctx().ok) return false; + uint32_t total_keys = history_len + new_len; + const uint32_t T = 128; + if (total_keys == 0 || M == 0 || (num_q_heads % num_kv_heads) != 0) return false; + uint32_t maxsc = (ring > 0u) ? ((total_keys > sink + ring) ? (sink + ring) : total_keys) : 256u; + if (maxsc > 7936u) return false; + bool mma2 = (ring == 0u && window == 0u && is_causal && head_dim == 512u && v_hdim == 512u + && num_q_heads == 8u && num_kv_heads == 1u); + bool hd256 = (ctx().psoAttnPreHd256 != nil && is_causal && head_dim == 256u && v_hdim == 256u + && num_q_heads == 8u && num_kv_heads == 1u); + if (mma2 || hd256) { + ensureEncoder(); + [g_enc setComputePipelineState:(hd256 ? ctx().psoAttnPreHd256 : ctx().psoAttnPreMma2)]; + setBufAt(q, (size_t)M*num_q_heads*head_dim*2, 0); + if (new_len > 0 && knew && vnew) { + setBufAt(knew, (size_t)new_len*num_kv_heads*head_dim*2, 1); + setBufAt(vnew, (size_t)new_len*num_kv_heads*v_hdim*2, 2); + } else { + [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + } + setCacheAt(kc, kc_bytes, 3); setCacheAt(vc, vc_bytes, 4); setCacheAt(ks, ks_bytes, 5); setCacheAt(vs, vs_bytes, 6); + setBufAt(out, (size_t)M*num_q_heads*v_hdim*2, 7); + [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; + [g_enc setBytes:&head_dim length:4 atIndex:10]; [g_enc setBytes:&v_hdim length:4 atIndex:11]; + [g_enc setBytes:&history_len length:4 atIndex:12];[g_enc setBytes:&scale length:4 atIndex:13]; + [g_enc setBytes:&q_pos0 length:4 atIndex:14]; [g_enc setBytes:&new_len length:4 atIndex:15]; + [g_enc setBytes:&M length:4 atIndex:16]; + if (hd256) { [g_enc setBytes:&sink length:4 atIndex:17]; [g_enc setBytes:&ring length:4 atIndex:18]; } + uint32_t mtiles = (M + 7u)/8u; + [g_enc dispatchThreadgroups:MTLSizeMake(mtiles, 1, 1) threadsPerThreadgroup:MTLSizeMake(hd256?256u:512u,1,1)]; + return true; + } + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoAttnPre]; + setBufAt(q, (size_t)M*num_q_heads*head_dim*2, 0); + if (new_len > 0 && knew && vnew) { + setBufAt(knew, (size_t)new_len*num_kv_heads*head_dim*2, 1); + setBufAt(vnew, (size_t)new_len*num_kv_heads*v_hdim*2, 2); + } else { + [g_enc setBuffer:ctx().dummy offset:0 atIndex:1]; [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + } + setCacheAt(kc, kc_bytes, 3); setCacheAt(vc, vc_bytes, 4); setCacheAt(ks, ks_bytes, 5); setCacheAt(vs, vs_bytes, 6); + setBufAt(out, (size_t)M*num_q_heads*v_hdim*2, 7); + [g_enc setBytes:&num_q_heads length:4 atIndex:8]; [g_enc setBytes:&num_kv_heads length:4 atIndex:9]; + [g_enc setBytes:&head_dim length:4 atIndex:10]; [g_enc setBytes:&v_hdim length:4 atIndex:11]; + [g_enc setBytes:&history_len length:4 atIndex:12];[g_enc setBytes:&scale length:4 atIndex:13]; + [g_enc setBytes:&q_pos0 length:4 atIndex:14]; [g_enc setBytes:&new_len length:4 atIndex:15]; + [g_enc setBytes:&window length:4 atIndex:16]; [g_enc setBytes:&is_causal length:4 atIndex:17]; + [g_enc setBytes:&maxsc length:4 atIndex:18]; + [g_enc setBytes:&sink length:4 atIndex:19]; [g_enc setBytes:&ring length:4 atIndex:20]; + [g_enc setThreadgroupMemoryLength:((size_t)maxsc + T)*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(M*num_q_heads,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + return true; +} + +bool cactus_metal_encode_binary_f32(int op, void* out, const void* a, const void* b, size_t n) { + if (!ctx().ok || !ctx().psoBinF32) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; + [g_enc setComputePipelineState:ctx().psoBinF32]; + setBufAt(a, n*4, 0); setBufAt(b, n*4, 1); setBufAt(out, n*4, 2); + [g_enc setBytes:&nn length:4 atIndex:3]; [g_enc setBytes:&o length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake((n+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_scalar_f32(int op, void* out, const void* in, size_t n, float p) { + if (!ctx().ok || !ctx().psoScaF32) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; + [g_enc setComputePipelineState:ctx().psoScaF32]; + setBufAt(in, n*4, 0); setBufAt(out, n*4, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; [g_enc setBytes:&p length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake((n+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_unary_f32(int op, void* out, const void* in, size_t n) { + if (!ctx().ok || !ctx().psoUnaF32) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; int o=op; + [g_enc setComputePipelineState:ctx().psoUnaF32]; + setBufAt(in, n*4, 0); setBufAt(out, n*4, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&o length:4 atIndex:3]; + [g_enc dispatchThreads:MTLSizeMake((n+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_clamp(void* out, const void* in, size_t n, float lo, float hi, int f32) { + id pso = f32 ? ctx().psoClampF32 : ctx().psoClampF16; + if (!ctx().ok || !pso) return false; + ensureEncoder(); + uint32_t nn=(uint32_t)n; size_t es = f32?4:2; + [g_enc setComputePipelineState:pso]; + setBufAt(in, n*es, 0); setBufAt(out, n*es, 1); + [g_enc setBytes:&nn length:4 atIndex:2]; [g_enc setBytes:&lo length:4 atIndex:3]; [g_enc setBytes:&hi length:4 atIndex:4]; + size_t grid = (n + 3) / 4; + [g_enc dispatchThreads:MTLSizeMake(grid,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_glu(void* out, const void* in, size_t split, size_t inner, size_t n_out) { + if (!ctx().ok || !ctx().psoGlu) return false; + ensureEncoder(); + uint32_t sp=(uint32_t)split, in_=(uint32_t)inner, nn=(uint32_t)n_out; + [g_enc setComputePipelineState:ctx().psoGlu]; + setBufAt(in, n_out*4, 0); setBufAt(out, n_out*2, 1); + [g_enc setBytes:&sp length:4 atIndex:2]; [g_enc setBytes:&in_ length:4 atIndex:3]; [g_enc setBytes:&nn length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n_out,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_layer_norm(void* out, const void* in, const void* w, const void* b, + size_t rows, size_t dim, float eps) { + if (!ctx().ok || !ctx().psoLayerNorm) return false; + ensureEncoder(); + uint32_t d=(uint32_t)dim, hb = b ? 1u : 0u; + uint32_t T = dim >= 1024 ? 1024u : 256u; + [g_enc setComputePipelineState:ctx().psoLayerNorm]; + setBufAt(in, rows*dim*2, 0); setBufAt(w, dim*2, 1); + if (b) setBufAt(b, dim*2, 2); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + setBufAt(out, rows*dim*2, 3); + [g_enc setBytes:&d length:4 atIndex:4]; [g_enc setBytes:&eps length:4 atIndex:5]; [g_enc setBytes:&hb length:4 atIndex:6]; + [g_enc setThreadgroupMemoryLength:2*T*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + return true; +} +bool cactus_metal_encode_softmax_rows(void* out, const void* in, size_t rows, size_t cols) { + if (!ctx().ok || !ctx().psoSoftmaxR) return false; + ensureEncoder(); + uint32_t c=(uint32_t)cols; + uint32_t T = cols >= 1024 ? 1024u : 256u; + [g_enc setComputePipelineState:ctx().psoSoftmaxR]; + setBufAt(in, rows*cols*2, 0); setBufAt(out, rows*cols*2, 1); + [g_enc setBytes:&c length:4 atIndex:2]; + [g_enc setThreadgroupMemoryLength:T*sizeof(float) atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(rows,1,1) threadsPerThreadgroup:MTLSizeMake(T,1,1)]; + return true; +} +bool cactus_metal_encode_conv1d_k3(void* out, const void* x, const void* w, int w_int8, + const void* w_scales, uint32_t w_gs, + uint32_t Cin, uint32_t L, uint32_t Cout, uint32_t Lout, uint32_t stride) { + if (!ctx().ok || !ctx().psoConv1dK3) return false; + ensureEncoder(); + id dq = nil; + if (w_int8) { + size_t rows = Cout, cols = (size_t)Cin*3; + dq = recycled(rows*cols*2); + if (!dq) return false; + __fp16* dst = (__fp16*)[dq contents]; + const int8_t* src = (const int8_t*)w; + if (w_gs == 0 || !w_scales) { + for (size_t i = 0; i < rows*cols; ++i) dst[i] = (__fp16)src[i]; + } else { + const __fp16* scales = (const __fp16*)w_scales; + size_t ngr = cols / w_gs; + for (size_t r = 0; r < rows; ++r) + for (size_t c = 0; c < cols; ++c) + dst[r*cols+c] = (__fp16)((float)src[r*cols+c] * (float)scales[r*ngr + c/w_gs]); + } + } + [g_enc setComputePipelineState:ctx().psoConv1dK3]; + setBufAt(x, (size_t)Cin*L*2, 0); + if (dq) [g_enc setBuffer:dq offset:0 atIndex:1]; else setBufAt(w, (size_t)Cout*Cin*3*2, 1); + setBufAt(out, (size_t)Cout*Lout*2, 2); + [g_enc setBytes:&Cin length:4 atIndex:3]; [g_enc setBytes:&L length:4 atIndex:4]; + [g_enc setBytes:&Lout length:4 atIndex:5]; [g_enc setBytes:&stride length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(Lout,Cout,1) threadsPerThreadgroup:MTLSizeMake(64,4,1)]; + return true; +} +static MPSMatrixMultiplication* mps_gemm_ab(uint32_t M, uint32_t N, uint32_t K, bool tr, float alpha, float beta) { + static std::map, MPSMatrixMultiplication*> cache; + uint32_t ab, bb; + std::memcpy(&ab, &alpha, 4); + std::memcpy(&bb, &beta, 4); + auto key = std::make_tuple(M,N,K,tr,((uint64_t)ab<<32)|bb); + auto it = cache.find(key); + if (it != cache.end()) return it->second; + MPSMatrixMultiplication* mm = [[MPSMatrixMultiplication alloc] + initWithDevice:ctx().dev transposeLeft:false transposeRight:tr + resultRows:M resultColumns:N interiorColumns:K alpha:alpha beta:beta]; + cache[key] = mm; + return mm; +} +static MPSMatrixMultiplication* mps_gemm_a(uint32_t M, uint32_t N, uint32_t K, bool tr, float alpha) { + return mps_gemm_ab(M, N, K, tr, alpha, 0.0f); +} +static MPSMatrixMultiplication* mps_gemm(uint32_t M, uint32_t N, uint32_t K, bool tr) { + return mps_gemm_a(M, N, K, tr, 1.0f); +} + +bool cactus_metal_encode_gemm_f16(void* out, const void* lhs, const void* rhs, + uint32_t M, uint32_t K, uint32_t N, int pretransposed) { + if (!ctx().ok) return false; + static const bool mps_ok = MPSSupportsMTLDevice(ctx().dev); + auto a = bufForPtrOff(lhs, (size_t)M*K*2); + auto b = bufForPtrOff(rhs, (size_t)N*K*2); + auto c = bufForPtrOff(out, (size_t)M*N*2); + int use_tr = pretransposed; + if (mps_ok && (a.second % 16)==0 && (b.second % 16)==0 && (c.second % 16)==0 + && (K % 8)==0 && (N % 8)==0) { + if (g_enc) { [g_enc endEncoding]; g_enc = nil; } + if (!g_cmd) g_cmd = [ctx().queue commandBuffer]; + MPSMatrixDescriptor* da = [MPSMatrixDescriptor matrixDescriptorWithRows:M columns:K + rowBytes:(size_t)K*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* db = use_tr + ? [MPSMatrixDescriptor matrixDescriptorWithRows:N columns:K rowBytes:(size_t)K*2 dataType:MPSDataTypeFloat16] + : [MPSMatrixDescriptor matrixDescriptorWithRows:K columns:N rowBytes:(size_t)N*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* dc = [MPSMatrixDescriptor matrixDescriptorWithRows:M columns:N + rowBytes:(size_t)N*2 dataType:MPSDataTypeFloat16]; + auto mat = [](id buf, size_t off, MPSMatrixDescriptor* d) -> MPSMatrix* { + auto key = std::make_tuple((__bridge void*)buf, off, (size_t)d.rows, (size_t)d.columns); + auto it = g_mps_mats.find(key); + if (it != g_mps_mats.end()) return it->second; + if (g_mps_mats.size() > 4096) g_mps_mats.clear(); + MPSMatrix* m = [[MPSMatrix alloc] initWithBuffer:buf offset:off descriptor:d]; + g_mps_mats[key] = m; + return m; + }; + MPSMatrix* ma = mat(a.first, a.second, da); + MPSMatrix* mb = mat(b.first, b.second, db); + MPSMatrix* mc = mat(c.first, c.second, dc); + [mps_gemm(M, N, K, use_tr != 0) encodeToCommandBuffer:g_cmd + leftMatrix:ma rightMatrix:mb resultMatrix:mc]; + return true; + } + if (!ctx().psoGdense || !ctx().psoStrided) return false; + ensureEncoder(); + id tbuf = nil; + if (!pretransposed) { + tbuf = recycled((size_t)K*N*2); + if (!tbuf) return false; + uint32_t oshape[2] = { N, K }, sstride[2] = { 1u, N }; + uint32_t total = N*K; + [g_enc setComputePipelineState:ctx().psoStrided]; + setBufAt(rhs, (size_t)K*N*2, 0); [g_enc setBuffer:tbuf offset:0 atIndex:1]; + [g_enc setBytes:oshape length:8 atIndex:2]; [g_enc setBytes:sstride length:8 atIndex:3]; + uint32_t nd=2, base=0; + [g_enc setBytes:&nd length:4 atIndex:4]; [g_enc setBytes:&total length:4 atIndex:5]; [g_enc setBytes:&base length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + } + [g_enc setComputePipelineState:ctx().psoGdense]; + setBufAt(lhs, (size_t)M*K*2, 0); + if (tbuf) [g_enc setBuffer:tbuf offset:0 atIndex:1]; else setBufAt(rhs, (size_t)N*K*2, 1); + setBufAt(out, (size_t)M*N*2, 2); + [g_enc setBytes:&K length:4 atIndex:3]; [g_enc setBytes:&N length:4 atIndex:4]; [g_enc setBytes:&M length:4 atIndex:5]; + [g_enc dispatchThreadgroups:MTLSizeMake((M+31)/32,(N+63)/64,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + return true; +} +bool cactus_metal_encode_attention_f16(void* out, const void* q, const void* k, const void* v, const void* mask, + uint32_t B, uint32_t T, uint32_t S, uint32_t HQ, uint32_t HKV, uint32_t D, uint32_t DV, + float scale, uint32_t causal, uint32_t pos_off, uint32_t window, float logit_cap, uint32_t mask_mode) { + if (!ctx().ok || !ctx().psoAttnDense) return false; + if (D > 128 || DV > 128 || (D % 4) != 0 || (DV % 4) != 0 || HKV == 0 || HQ % HKV != 0) return false; + static const bool mps_attn_ok = MPSSupportsMTLDevice(ctx().dev); + bool mask_ok = mask_mode == 0 || (mask_mode == 2 && mask && ctx().psoMaskFill); + if (ctx().psoAttnFlash && B == 1 && HQ == HKV && !causal && window == 0 + && logit_cap == 0.0f && D == 64 && DV == 64 && S >= 256 + && (mask_mode == 0 || (mask_mode == 2 && mask))) { + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoAttnFlash]; + setBufAt(q, (size_t)T*HQ*D*2, 0); setBufAt(k, (size_t)S*HKV*D*2, 1); + setBufAt(v, (size_t)S*HKV*DV*2, 2); setBufAt(out, (size_t)T*HQ*DV*2, 3); + if (mask_mode == 2 && mask) setBufAt(mask, (size_t)T*S*2, 4); + else [g_enc setBuffer:ctx().dummy offset:0 atIndex:4]; + [g_enc setBytes:&T length:4 atIndex:5]; [g_enc setBytes:&S length:4 atIndex:6]; + [g_enc setBytes:&HQ length:4 atIndex:7]; [g_enc setBytes:&scale length:4 atIndex:8]; + [g_enc setBytes:&mask_mode length:4 atIndex:9]; + [g_enc dispatchThreadgroups:MTLSizeMake((T + 63) / 64, HQ, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)]; + return true; + } + if (mps_attn_ok && B == 1 && HQ == HKV && !causal && mask_ok && window == 0 + && logit_cap == 0.0f && S >= 512 && (S % 8) == 0 && (D % 8) == 0 && (DV % 8) == 0 + && ctx().psoSoftmaxR) { + auto qb = bufForPtrOff(q, (size_t)T*HQ*D*2); + auto kb = bufForPtrOff(k, (size_t)S*HKV*D*2); + auto vb = bufForPtrOff(v, (size_t)S*HKV*DV*2); + auto ob = bufForPtrOff(out, (size_t)T*HQ*DV*2); + bool aligned = (qb.second % 16)==0 && (kb.second % 16)==0 && (vb.second % 16)==0 && (ob.second % 16)==0 + && ((D*2) % 16)==0 && ((DV*2) % 16)==0; + if (aligned) { + size_t need = (size_t)HQ*T*S*2; + if (!g_attn_scores || g_attn_scores.length < need) + g_attn_scores = owned_shared(need); + id sbuf = g_attn_scores; + if (sbuf) { + float beta = 0.0f; + if (mask_mode == 2) { + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoMaskFill]; + setBufAt(mask, (size_t)T*S*2, 0); + [g_enc setBuffer:sbuf offset:0 atIndex:1]; + uint32_t ts = T*S; + [g_enc setBytes:&ts length:4 atIndex:2]; + [g_enc dispatchThreads:MTLSizeMake(ts, HQ, 1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + beta = 1.0f; + } + if (g_enc) { [g_enc endEncoding]; g_enc = nil; } + if (!g_cmd) g_cmd = [ctx().queue commandBuffer]; + MPSMatrixDescriptor* dq = [MPSMatrixDescriptor matrixDescriptorWithRows:T columns:D + rowBytes:(size_t)HQ*D*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* dk = [MPSMatrixDescriptor matrixDescriptorWithRows:S columns:D + rowBytes:(size_t)HKV*D*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* ds = [MPSMatrixDescriptor matrixDescriptorWithRows:T columns:S + rowBytes:(size_t)S*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* dv = [MPSMatrixDescriptor matrixDescriptorWithRows:S columns:DV + rowBytes:(size_t)HKV*DV*2 dataType:MPSDataTypeFloat16]; + MPSMatrixDescriptor* do2 = [MPSMatrixDescriptor matrixDescriptorWithRows:T columns:DV + rowBytes:(size_t)HQ*DV*2 dataType:MPSDataTypeFloat16]; + MPSMatrixMultiplication* mqk = mps_gemm_ab(T, S, D, true, scale, beta); + for (uint32_t h = 0; h < HQ; ++h) { + MPSMatrix* mq = [[MPSMatrix alloc] initWithBuffer:qb.first offset:qb.second + (size_t)h*D*2 descriptor:dq]; + MPSMatrix* mk = [[MPSMatrix alloc] initWithBuffer:kb.first offset:kb.second + (size_t)h*D*2 descriptor:dk]; + MPSMatrix* msc = [[MPSMatrix alloc] initWithBuffer:sbuf offset:(size_t)h*T*S*2 descriptor:ds]; + [mqk encodeToCommandBuffer:g_cmd leftMatrix:mq rightMatrix:mk resultMatrix:msc]; + } + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoSoftmaxR]; + [g_enc setBuffer:sbuf offset:0 atIndex:0]; + [g_enc setBuffer:sbuf offset:0 atIndex:1]; + uint32_t cols = S; + [g_enc setBytes:&cols length:4 atIndex:2]; + [g_enc setThreadgroupMemoryLength:256*4 atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake((size_t)HQ*T,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + [g_enc endEncoding]; g_enc = nil; + MPSMatrixMultiplication* mpv = mps_gemm(T, DV, S, false); + for (uint32_t h = 0; h < HQ; ++h) { + MPSMatrix* msc = [[MPSMatrix alloc] initWithBuffer:sbuf offset:(size_t)h*T*S*2 descriptor:ds]; + MPSMatrix* mv = [[MPSMatrix alloc] initWithBuffer:vb.first offset:vb.second + (size_t)h*DV*2 descriptor:dv]; + MPSMatrix* mo = [[MPSMatrix alloc] initWithBuffer:ob.first offset:ob.second + (size_t)h*DV*2 descriptor:do2]; + [mpv encodeToCommandBuffer:g_cmd leftMatrix:msc rightMatrix:mv resultMatrix:mo]; + } + return true; + } + } + } + ensureEncoder(); + struct { uint32_t T,S,HQ,HKV,D,DV; float scale; uint32_t causal,pos_off,window; float logit_cap; uint32_t mask_mode; } U = + { T,S,HQ,HKV,D,DV, scale, causal,pos_off,window, logit_cap, mask_mode }; + bool d64 = (D <= 64 && DV <= 64 && ctx().psoAttnD64); + [g_enc setComputePipelineState:(d64 ? ctx().psoAttnD64 : ctx().psoAttnDense)]; + setBufAt(q, (size_t)B*T*HQ*D*2, 0); setBufAt(k, (size_t)B*S*HKV*D*2, 1); + setBufAt(v, (size_t)B*S*HKV*DV*2, 2); setBufAt(out, (size_t)B*T*HQ*DV*2, 3); + if (mask_mode && mask) { + size_t mb = (mask_mode>=3u) ? (size_t)B*HQ*T*S*2 : (size_t)B*T*S*2; + setBufAt(mask, mb, 4); + } else [g_enc setBuffer:ctx().dummy offset:0 atIndex:4]; + [g_enc setBytes:&U length:sizeof(U) atIndex:5]; + [g_enc dispatchThreadgroups:MTLSizeMake(d64 ? (T+7)/8 : (T+3)/4,HQ,B) threadsPerThreadgroup:MTLSizeMake(32,1,1)]; + return true; +} + +bool cactus_metal_encode_reduce_axis(int op, void* out, const void* in, uint32_t outer, + uint32_t axis_size, uint32_t inner, int f32) { + if (!ctx().ok || !ctx().psoReduceF16 || !ctx().psoReduceF32) return false; + ensureEncoder(); + size_t es = f32 ? 4 : 2; + uint32_t n = outer * inner; + [g_enc setComputePipelineState:(f32 ? ctx().psoReduceF32 : ctx().psoReduceF16)]; + setBufAt(in, (size_t)outer*axis_size*inner*es, 0); setBufAt(out, (size_t)n*es, 1); + [g_enc setBytes:&axis_size length:4 atIndex:2]; [g_enc setBytes:&inner length:4 atIndex:3]; + [g_enc setBytes:&n length:4 atIndex:4]; [g_enc setBytes:&op length:4 atIndex:5]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_cumsum(void* out, const void* in, uint32_t outer, + uint32_t axis_size, uint32_t inner, int f32) { + if (!ctx().ok || !ctx().psoCumsumF16 || !ctx().psoCumsumF32) return false; + ensureEncoder(); + size_t es = f32 ? 4 : 2; + uint32_t n = outer * inner; + size_t bytes = (size_t)outer*axis_size*inner*es; + [g_enc setComputePipelineState:(f32 ? ctx().psoCumsumF32 : ctx().psoCumsumF16)]; + setBufAt(in, bytes, 0); setBufAt(out, bytes, 1); + [g_enc setBytes:&axis_size length:4 atIndex:2]; [g_enc setBytes:&inner length:4 atIndex:3]; + [g_enc setBytes:&n length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_concat2(void* out, const void* a, const void* b, + uint32_t a_outer, uint32_t b_outer, + uint32_t a_axis, uint32_t b_axis, uint32_t inner) { + if (!ctx().ok || !ctx().psoConcat2 || a_outer == 0 || b_outer == 0) return false; + ensureEncoder(); + uint32_t outer = a_outer > b_outer ? a_outer : b_outer; + uint32_t n = outer * (a_axis + b_axis) * inner; + [g_enc setComputePipelineState:ctx().psoConcat2]; + setBufAt(a, (size_t)a_outer*a_axis*inner*2, 0); setBufAt(b, (size_t)b_outer*b_axis*inner*2, 1); + setBufAt(out, (size_t)n*2, 2); + [g_enc setBytes:&a_axis length:4 atIndex:3]; [g_enc setBytes:&b_axis length:4 atIndex:4]; + [g_enc setBytes:&inner length:4 atIndex:5]; [g_enc setBytes:&n length:4 atIndex:6]; + [g_enc setBytes:&a_outer length:4 atIndex:7]; [g_enc setBytes:&b_outer length:4 atIndex:8]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_gather_f32idx(void* out, const void* table, const void* idx, + uint32_t rows, uint32_t D, size_t table_bytes) { + if (!ctx().ok || !ctx().psoGatherF32Idx) return false; + ensureEncoder(); + uint32_t n = rows * D; + [g_enc setComputePipelineState:ctx().psoGatherF32Idx]; + setBufAt(table, table_bytes, 0); setBufAt(idx, (size_t)rows*4, 1); setBufAt(out, (size_t)n*2, 2); + [g_enc setBytes:&D length:4 atIndex:3]; [g_enc setBytes:&n length:4 atIndex:4]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_rope_full(void* out, const void* in, uint32_t tokens, uint32_t S, + uint32_t H, uint32_t D, uint32_t rot, uint32_t pos0, + float theta, int gptj) { + if (!ctx().ok || !ctx().psoRopeFull) return false; + ensureEncoder(); + uint32_t gj = gptj ? 1u : 0u; + [g_enc setComputePipelineState:ctx().psoRopeFull]; + setBufAt(in, (size_t)tokens*D*2, 0); setBufAt(out, (size_t)tokens*D*2, 1); + [g_enc setBytes:&S length:4 atIndex:2]; [g_enc setBytes:&H length:4 atIndex:3]; + [g_enc setBytes:&D length:4 atIndex:4]; [g_enc setBytes:&rot length:4 atIndex:5]; + [g_enc setBytes:&pos0 length:4 atIndex:6]; [g_enc setBytes:&theta length:4 atIndex:7]; + [g_enc setBytes:&gj length:4 atIndex:8]; + uint32_t gx = rot/2 + (D > rot ? D - rot : 0); + if (gx == 0) gx = 1; + [g_enc dispatchThreads:MTLSizeMake(gx, tokens, 1) threadsPerThreadgroup:MTLSizeMake(std::min(gx, 64u), 4, 1)]; + return true; +} +bool cactus_metal_encode_maxpool1d(void* out, const void* in, uint32_t NC, uint32_t L, + uint32_t Lout, uint32_t K, uint32_t stride) { + if (!ctx().ok || !ctx().psoMaxpool1d) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoMaxpool1d]; + setBufAt(in, (size_t)NC*L*2, 0); setBufAt(out, (size_t)NC*Lout*2, 1); + [g_enc setBytes:&L length:4 atIndex:2]; [g_enc setBytes:&Lout length:4 atIndex:3]; + [g_enc setBytes:&K length:4 atIndex:4]; [g_enc setBytes:&stride length:4 atIndex:5]; + [g_enc dispatchThreads:MTLSizeMake(Lout, NC, 1) threadsPerThreadgroup:MTLSizeMake(std::min(Lout, 64u), 4, 1)]; + return true; +} +bool cactus_metal_encode_bilinear(void* out, const void* in, uint32_t sh, uint32_t sw, + uint32_t dh, uint32_t dw, uint32_t E, int align) { + if (!ctx().ok || !ctx().psoBilinear) return false; + ensureEncoder(); + uint32_t al = align ? 1u : 0u; + [g_enc setComputePipelineState:ctx().psoBilinear]; + setBufAt(in, (size_t)sh*sw*E*2, 0); setBufAt(out, (size_t)dh*dw*E*2, 1); + [g_enc setBytes:&sh length:4 atIndex:2]; [g_enc setBytes:&sw length:4 atIndex:3]; + [g_enc setBytes:&dh length:4 atIndex:4]; [g_enc setBytes:&dw length:4 atIndex:5]; + [g_enc setBytes:&E length:4 atIndex:6]; [g_enc setBytes:&al length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake(E, dh*dw, 1) threadsPerThreadgroup:MTLSizeMake(std::min(E, 64u), 4, 1)]; + return true; +} +bool cactus_metal_encode_conv1d_gen(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t Cin, uint32_t L, uint32_t Cout, + uint32_t Lout, uint32_t K, uint32_t stride, int w_ck_co) { + if (!ctx().ok || !ctx().psoConv1dGen) return false; + ensureEncoder(); + uint32_t hb = bias ? 1u : 0u, wl = w_ck_co ? 1u : 0u; + [g_enc setComputePipelineState:ctx().psoConv1dGen]; + setBufAt(x, (size_t)N*Cin*L*2, 0); setBufAt(w, (size_t)Cout*Cin*K*2, 1); + if (bias) setBufAt(bias, (size_t)Cout*2, 2); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + setBufAt(out, (size_t)N*Cout*Lout*2, 3); + [g_enc setBytes:&Cin length:4 atIndex:4]; [g_enc setBytes:&L length:4 atIndex:5]; + [g_enc setBytes:&Cout length:4 atIndex:6]; [g_enc setBytes:&Lout length:4 atIndex:7]; + [g_enc setBytes:&K length:4 atIndex:8]; [g_enc setBytes:&stride length:4 atIndex:9]; + [g_enc setBytes:&hb length:4 atIndex:10]; [g_enc setBytes:&wl length:4 atIndex:11]; + [g_enc dispatchThreads:MTLSizeMake(Lout, Cout, N) threadsPerThreadgroup:MTLSizeMake(std::min(Lout, 64u), 4, 1)]; + return true; +} +bool cactus_metal_encode_conv1d_nlc_dw(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t L, uint32_t C, uint32_t K, + uint32_t dil, uint32_t pad) { + if (!ctx().ok || !ctx().psoConv1dNlcDw) return false; + ensureEncoder(); + uint32_t hb = bias ? 1u : 0u; + [g_enc setComputePipelineState:ctx().psoConv1dNlcDw]; + setBufAt(x, (size_t)N*L*C*2, 0); setBufAt(w, (size_t)C*K*2, 1); + if (bias) setBufAt(bias, (size_t)C*2, 2); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + setBufAt(out, (size_t)N*L*C*2, 3); + [g_enc setBytes:&L length:4 atIndex:4]; [g_enc setBytes:&C length:4 atIndex:5]; + [g_enc setBytes:&K length:4 atIndex:6]; [g_enc setBytes:&dil length:4 atIndex:7]; + [g_enc setBytes:&pad length:4 atIndex:8]; [g_enc setBytes:&hb length:4 atIndex:9]; + [g_enc dispatchThreads:MTLSizeMake(C, L, N) threadsPerThreadgroup:MTLSizeMake(std::min(C, 64u), 4, 1)]; + return true; +} +bool cactus_metal_encode_conv2d(void* out, const void* x, const void* w, const void* bias, + uint32_t N, uint32_t Cin, uint32_t H, uint32_t W, uint32_t Cout, + uint32_t Ho, uint32_t Wo, uint32_t K, uint32_t stride, + uint32_t pad, int dw) { + if (!ctx().ok || !ctx().psoConv2d) return false; + ensureEncoder(); + uint32_t hb = bias ? 1u : 0u, dwf = dw ? 1u : 0u; + size_t wbytes = dw ? (size_t)Cout*K*K*2 : (size_t)Cout*Cin*K*K*2; + [g_enc setComputePipelineState:ctx().psoConv2d]; + setBufAt(x, (size_t)N*Cin*H*W*2, 0); setBufAt(w, wbytes, 1); + if (bias) setBufAt(bias, (size_t)Cout*2, 2); else [g_enc setBuffer:ctx().dummy offset:0 atIndex:2]; + setBufAt(out, (size_t)N*Cout*Ho*Wo*2, 3); + [g_enc setBytes:&Cin length:4 atIndex:4]; [g_enc setBytes:&H length:4 atIndex:5]; + [g_enc setBytes:&W length:4 atIndex:6]; [g_enc setBytes:&Cout length:4 atIndex:7]; + [g_enc setBytes:&Ho length:4 atIndex:8]; [g_enc setBytes:&Wo length:4 atIndex:9]; + [g_enc setBytes:&K length:4 atIndex:10]; [g_enc setBytes:&stride length:4 atIndex:11]; + [g_enc setBytes:&pad length:4 atIndex:12]; [g_enc setBytes:&dwf length:4 atIndex:13]; + [g_enc setBytes:&hb length:4 atIndex:14]; + uint32_t wq = (Wo + 3) / 4; + [g_enc dispatchThreads:MTLSizeMake(wq, Ho, (size_t)N*Cout) threadsPerThreadgroup:MTLSizeMake(std::min(wq, 32u), std::min(Ho, 8u), 1)]; + return true; +} +bool cactus_metal_encode_batchnorm(void* out, const void* x, const void* w, const void* b, + const void* rm, const void* rv, uint32_t C, uint32_t inner, + uint32_t total, float eps) { + if (!ctx().ok || !ctx().psoBatchnorm) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoBatchnorm]; + setBufAt(x, (size_t)total*2, 0); setBufAt(w, (size_t)C*2, 1); setBufAt(b, (size_t)C*2, 2); + setBufAt(rm, (size_t)C*2, 3); setBufAt(rv, (size_t)C*2, 4); setBufAt(out, (size_t)total*2, 5); + [g_enc setBytes:&C length:4 atIndex:6]; [g_enc setBytes:&inner length:4 atIndex:7]; + [g_enc setBytes:&eps length:4 atIndex:8]; [g_enc setBytes:&total length:4 atIndex:9]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_groupnorm(void* out, const void* x, const void* w, const void* b, + uint32_t N, uint32_t C, uint32_t S, uint32_t groups, float eps) { + if (!ctx().ok || !ctx().psoGroupnorm || groups == 0 || C % groups != 0) return false; + ensureEncoder(); + uint32_t cpg = C / groups; + [g_enc setComputePipelineState:ctx().psoGroupnorm]; + setBufAt(x, (size_t)N*C*S*2, 0); setBufAt(w, (size_t)C*2, 1); setBufAt(b, (size_t)C*2, 2); + setBufAt(out, (size_t)N*C*S*2, 3); + [g_enc setBytes:&cpg length:4 atIndex:4]; [g_enc setBytes:&S length:4 atIndex:5]; + [g_enc setBytes:&C length:4 atIndex:6]; [g_enc setBytes:&eps length:4 atIndex:7]; + [g_enc setThreadgroupMemoryLength:256*4 atIndex:0]; + [g_enc dispatchThreadgroups:MTLSizeMake(groups, N, 1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_bias_add_rows(void* y, const void* bias, uint32_t C, uint32_t total) { + if (!ctx().ok || !ctx().psoBiasAddRows) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoBiasAddRows]; + setBufAt(y, (size_t)total*2, 0); setBufAt(bias, (size_t)C*2, 1); + [g_enc setBytes:&C length:4 atIndex:2]; [g_enc setBytes:&total length:4 atIndex:3]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_elemwise_chain(void* out, const void* in, const float* steps, + uint32_t nsteps, const void* side0, const void* side1, + const void* side2, const size_t* side_elems, + size_t n, uint32_t flags, uint32_t inner) { + if (!ctx().ok || !ctx().psoEwChain || nsteps == 0 || nsteps > 12) return false; + ensureEncoder(); + uint32_t ns = nsteps, nn = (uint32_t)n; + size_t ein = (flags & 1u) ? 4 : 2, eout = (flags & 2u) ? 4 : 2; + [g_enc setComputePipelineState:ctx().psoEwChain]; + setBufAt(in, n*ein, 0); setBufAt(out, n*eout, 1); + [g_enc setBytes:steps length:nsteps*16 atIndex:2]; + [g_enc setBytes:&ns length:4 atIndex:3]; + setBufAt(side0 ? side0 : in, (side0 ? side_elems[0] : n)*((flags & 4u) ? 4 : 2), 4); + setBufAt(side1 ? side1 : in, (side1 ? side_elems[1] : n)*((flags & 8u) ? 4 : 2), 5); + setBufAt(side2 ? side2 : in, (side2 ? side_elems[2] : n)*((flags & 16u) ? 4 : 2), 6); + [g_enc setBytes:&nn length:4 atIndex:7]; + [g_enc setBytes:&flags length:4 atIndex:8]; + uint32_t innr = inner; + [g_enc setBytes:&innr length:4 atIndex:9]; + [g_enc dispatchThreads:MTLSizeMake((nn+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} + +bool cactus_metal_encode_rms_norm_add_rows(void* ysum, void* ynorm, const void* x, const void* res, + const void* w, uint32_t rows, uint32_t dim, float eps, + int clipped) { + if (!ctx().ok || !ctx().psoRmsAddSimd) return false; + ensureEncoder(); + uint32_t cl = clipped ? 1u : 0u; + [g_enc setComputePipelineState:ctx().psoRmsAddSimd]; + setBufAt(x, (size_t)rows*dim*2, 0); setBufAt(res, (size_t)rows*dim*2, 1); + setBufAt(w, (size_t)dim*2, 2); setBufAt(ysum, (size_t)rows*dim*2, 3); + setBufAt(ynorm, (size_t)rows*dim*2, 4); + [g_enc setBytes:&dim length:4 atIndex:5]; [g_enc setBytes:&eps length:4 atIndex:6]; + [g_enc setBytes:&rows length:4 atIndex:7]; [g_enc setBytes:&cl length:4 atIndex:8]; + [g_enc dispatchThreadgroups:MTLSizeMake((rows+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + return true; +} +bool cactus_metal_encode_gemm_batch(void* out, const void* a, const void* b, + uint32_t M, uint32_t K, uint32_t N, uint32_t batch, int f32out, int f32a) { + if (!ctx().ok || !ctx().psoGemmBatch || !ctx().psoGemmBatchF32A) return false; + ensureEncoder(); + uint32_t fo = f32out ? 1u : 0u; + [g_enc setComputePipelineState:(f32a ? ctx().psoGemmBatchF32A : ctx().psoGemmBatch)]; + setBufAt(a, (size_t)batch*M*K*(f32a?4:2), 0); setBufAt(b, (size_t)batch*K*N*2, 1); + setBufAt(out, (size_t)batch*M*N*(f32out?4:2), 2); + [g_enc setBytes:&M length:4 atIndex:3]; [g_enc setBytes:&K length:4 atIndex:4]; + [g_enc setBytes:&N length:4 atIndex:5]; [g_enc setBytes:&fo length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake((size_t)M*N, batch, 1) threadsPerThreadgroup:MTLSizeMake(64,4,1)]; + return true; +} +bool cactus_metal_encode_conv1d_dw(void* out, const void* x, const void* w, + uint32_t C, uint32_t L, uint32_t Lout, uint32_t K, uint32_t stride) { + if (!ctx().ok || !ctx().psoConvDw) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoConvDw]; + setBufAt(x, (size_t)C*L*2, 0); setBufAt(w, (size_t)C*K*2, 1); setBufAt(out, (size_t)C*Lout*2, 2); + [g_enc setBytes:&L length:4 atIndex:3]; [g_enc setBytes:&Lout length:4 atIndex:4]; + [g_enc setBytes:&K length:4 atIndex:5]; [g_enc setBytes:&stride length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(Lout, C, 1) threadsPerThreadgroup:MTLSizeMake(64,4,1)]; + return true; +} +bool cactus_metal_encode_transpose2d(void* out, const void* in, uint32_t batch, uint32_t R, uint32_t C) { + if (!ctx().ok || !ctx().psoTranspose2d) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoTranspose2d]; + setBufAt(in, (size_t)batch*R*C*2, 0); setBufAt(out, (size_t)batch*R*C*2, 1); + [g_enc setBytes:&R length:4 atIndex:2]; [g_enc setBytes:&C length:4 atIndex:3]; + [g_enc dispatchThreadgroups:MTLSizeMake((C+31)/32, (R+31)/32, batch) threadsPerThreadgroup:MTLSizeMake(32,8,1)]; + return true; +} +bool cactus_metal_encode_strided_copy(void* out, const void* in, const uint32_t* oshape, + const uint32_t* sstride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes) { + if (!ctx().ok || ndim == 0 || ndim > 8) return false; + ensureEncoder(); + if (ndim >= 2 && sstride[ndim-1] == 1 && (oshape[ndim-1] % 4) == 0 && ctx().psoStridedRows) { + uint32_t inner = oshape[ndim-1]; + uint32_t inner4 = inner / 4; + uint32_t nd2 = ndim - 1; + uint32_t rows = total / inner; + [g_enc setComputePipelineState:ctx().psoStridedRows]; + setBufAt(in, in_bytes, 0); setBufAt(out, out_bytes, 1); + [g_enc setBytes:oshape length:nd2*4 atIndex:2]; [g_enc setBytes:sstride length:nd2*4 atIndex:3]; + [g_enc setBytes:&nd2 length:4 atIndex:4]; [g_enc setBytes:&rows length:4 atIndex:5]; + [g_enc setBytes:&base length:4 atIndex:6]; [g_enc setBytes:&inner4 length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake(inner4, rows, 1) threadsPerThreadgroup:MTLSizeMake(std::min(inner4, 64u), 4, 1)]; + return true; + } + [g_enc setComputePipelineState:ctx().psoStrided]; + setBufAt(in, in_bytes, 0); setBufAt(out, out_bytes, 1); + [g_enc setBytes:oshape length:ndim*4 atIndex:2]; [g_enc setBytes:sstride length:ndim*4 atIndex:3]; + [g_enc setBytes:&ndim length:4 atIndex:4]; [g_enc setBytes:&total length:4 atIndex:5]; [g_enc setBytes:&base length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_bcast_binary(int op, void* out, const void* a, const void* b, + const uint32_t* oshape, const uint32_t* astride, const uint32_t* bstride, uint32_t ndim, uint32_t total, + size_t a_bytes, size_t b_bytes, size_t out_bytes) { + if (!ctx().ok || ndim == 0 || ndim > 8) return false; + ensureEncoder(); + int o = op; + if (ndim >= 2 && oshape[ndim-1] > 1 && ctx().psoBcastRows) { + uint32_t inner = oshape[ndim-1]; + uint32_t ainner = astride[ndim-1], binner = bstride[ndim-1]; + uint32_t nd2 = ndim - 1; + uint32_t rows = total / inner; + [g_enc setComputePipelineState:ctx().psoBcastRows]; + setBufAt(a, a_bytes, 0); setBufAt(b, b_bytes, 1); setBufAt(out, out_bytes, 2); + [g_enc setBytes:oshape length:nd2*4 atIndex:3]; [g_enc setBytes:astride length:nd2*4 atIndex:4]; + [g_enc setBytes:bstride length:nd2*4 atIndex:5]; + [g_enc setBytes:&nd2 length:4 atIndex:6]; [g_enc setBytes:&rows length:4 atIndex:7]; + [g_enc setBytes:&o length:4 atIndex:8]; [g_enc setBytes:&inner length:4 atIndex:9]; + [g_enc setBytes:&ainner length:4 atIndex:10]; [g_enc setBytes:&binner length:4 atIndex:11]; + [g_enc dispatchThreads:MTLSizeMake(inner, rows, 1) threadsPerThreadgroup:MTLSizeMake(std::min(inner, 64u), 4, 1)]; + return true; + } + [g_enc setComputePipelineState:ctx().psoBcast]; + setBufAt(a, a_bytes, 0); setBufAt(b, b_bytes, 1); setBufAt(out, out_bytes, 2); + [g_enc setBytes:oshape length:ndim*4 atIndex:3]; [g_enc setBytes:astride length:ndim*4 atIndex:4]; [g_enc setBytes:bstride length:ndim*4 atIndex:5]; + [g_enc setBytes:&ndim length:4 atIndex:6]; [g_enc setBytes:&total length:4 atIndex:7]; [g_enc setBytes:&o length:4 atIndex:8]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +bool cactus_metal_encode_strided_scatter(void* out, const void* in, const uint32_t* ishape, + const uint32_t* ostride, uint32_t ndim, uint32_t total, uint32_t base, size_t in_bytes, size_t out_bytes) { + if (!ctx().ok || ndim == 0 || ndim > 8) return false; + ensureEncoder(); + if (ndim >= 2 && ostride[ndim-1] == 1 && (ishape[ndim-1] % 4) == 0 && ctx().psoScatterRows) { + uint32_t inner = ishape[ndim-1]; + uint32_t inner4 = inner / 4; + uint32_t nd2 = ndim - 1; + uint32_t rows = total / inner; + [g_enc setComputePipelineState:ctx().psoScatterRows]; + setBufAt(in, in_bytes, 0); setBufAt(out, out_bytes, 1); + [g_enc setBytes:ishape length:nd2*4 atIndex:2]; [g_enc setBytes:ostride length:nd2*4 atIndex:3]; + [g_enc setBytes:&nd2 length:4 atIndex:4]; [g_enc setBytes:&rows length:4 atIndex:5]; + [g_enc setBytes:&base length:4 atIndex:6]; [g_enc setBytes:&inner4 length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake(inner4, rows, 1) threadsPerThreadgroup:MTLSizeMake(std::min(inner4, 64u), 4, 1)]; + return true; + } + [g_enc setComputePipelineState:ctx().psoScatter]; + setBufAt(in, in_bytes, 0); setBufAt(out, out_bytes, 1); + [g_enc setBytes:ishape length:ndim*4 atIndex:2]; [g_enc setBytes:ostride length:ndim*4 atIndex:3]; + [g_enc setBytes:&ndim length:4 atIndex:4]; [g_enc setBytes:&total length:4 atIndex:5]; [g_enc setBytes:&base length:4 atIndex:6]; + [g_enc dispatchThreads:MTLSizeMake(total,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; + return true; +} +static bool encode_kv_append(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, + size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + if (!ctx().ok) return false; + uint32_t num_groups = (hdim + group_size - 1)/group_size, n = M*kv_heads*num_groups; + ensureEncoder(); + [g_enc setComputePipelineState:(M == 1 ? ctx().psoKvAppend : ctx().psoKvAppendM)]; + setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); + [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; + [g_enc setBytes:¤t_len length:4 atIndex:5]; [g_enc setBytes:&group_size length:4 atIndex:6]; + if (M != 1) [g_enc setBytes:&M length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; + return true; +} + +bool cactus_metal_encode_kv_append_i8(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, + size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + return encode_kv_append(src, int8base, scalebase, kv_heads, hdim, current_len, group_size, 1, + src_bytes, int8_bytes, scale_bytes); +} + +bool cactus_metal_encode_kv_append_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, + size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + return encode_kv_append(src, int8base, scalebase, kv_heads, hdim, current_len, group_size, M, + src_bytes, int8_bytes, scale_bytes); +} + +bool cactus_metal_encode_gemv_bias(void* out, const void* x, const void* w, const void* bias, + uint32_t K, uint32_t N, int pretransposed) { + if (!ctx().ok || !ctx().psoGemvBias || !bias) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoGemvBias]; + setBufAt(x, (size_t)K*2, 0); + setBufAt(w, (size_t)K*N*2, 1); + setBufAt(bias, (size_t)N*2, 2); + setBufAt(out, (size_t)N*2, 3); + uint32_t trv = pretransposed ? 1u : 0u; + [g_enc setBytes:&K length:4 atIndex:4]; [g_enc setBytes:&N length:4 atIndex:5]; + [g_enc setBytes:&trv length:4 atIndex:6]; + [g_enc dispatchThreadgroups:MTLSizeMake((N+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; + return true; +} + +bool cactus_metal_encode_rel_pos_bias(void* y, const void* q, const void* r, + uint32_t B, uint32_t T, uint32_t H, uint32_t D, uint32_t R, int r_batched, float scale) { + if (!ctx().ok || !ctx().psoRelPosBias) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoRelPosBias]; + setBufAt(q, (size_t)B*T*H*D*2, 0); + setBufAt(r, (size_t)(r_batched?B:1)*R*H*D*2, 1); + setBufAt(y, (size_t)B*H*T*T*2, 2); + uint32_t rbs = r_batched ? R*H*D : 0u; + [g_enc setBytes:&T length:4 atIndex:3]; [g_enc setBytes:&H length:4 atIndex:4]; + [g_enc setBytes:&D length:4 atIndex:5]; [g_enc setBytes:&rbs length:4 atIndex:6]; + [g_enc setBytes:&scale length:4 atIndex:7]; + [g_enc dispatchThreads:MTLSizeMake((size_t)T*T, H, B) + threadsPerThreadgroup:MTLSizeMake(64, H < 4 ? H : 4, 1)]; + return true; +} + +bool cactus_metal_encode_conv_cache_append(void* out, const void* src, void* ring, + uint32_t hd, uint32_t ws, uint32_t nnew, uint32_t head0, uint32_t count_new, + uint32_t num_rows, int src_f32) { + if (!ctx().ok || !ctx().psoConvCacheAppend) return false; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoConvCacheAppend]; + setBufAt(src, (size_t)num_rows*hd*(src_f32?4:2), 0); + setBufAt(ring, (size_t)ws*hd*2, 1); + setBufAt(out, (size_t)ws*hd*2, 2); + uint32_t f32 = src_f32 ? 1u : 0u; + [g_enc setBytes:&hd length:4 atIndex:3]; [g_enc setBytes:&ws length:4 atIndex:4]; + [g_enc setBytes:&nnew length:4 atIndex:5]; [g_enc setBytes:&head0 length:4 atIndex:6]; + [g_enc setBytes:&count_new length:4 atIndex:7]; [g_enc setBytes:&num_rows length:4 atIndex:8]; + [g_enc setBytes:&f32 length:4 atIndex:9]; + [g_enc dispatchThreads:MTLSizeMake(hd, ws + nnew, 1) + threadsPerThreadgroup:MTLSizeMake(hd < 64 ? hd : 64, 4, 1)]; + return true; +} + +bool cactus_metal_encode_kv_append_ring_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t current_len, uint32_t group_size, uint32_t M, + uint32_t sink, uint32_t W, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + if (!ctx().ok) return false; + uint32_t num_groups = (hdim + group_size - 1)/group_size, n = M*kv_heads*num_groups; + ensureEncoder(); + [g_enc setComputePipelineState:ctx().psoKvAppendRingM]; + setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); + [g_enc setBytes:&kv_heads length:4 atIndex:3]; [g_enc setBytes:&hdim length:4 atIndex:4]; + [g_enc setBytes:¤t_len length:4 atIndex:5]; [g_enc setBytes:&group_size length:4 atIndex:6]; + [g_enc setBytes:&M length:4 atIndex:7]; + [g_enc setBytes:&sink length:4 atIndex:8]; [g_enc setBytes:&W length:4 atIndex:9]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; + return true; +} + +static bool encode_kv_append_sliding(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, + uint32_t group_size, uint32_t M, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + if (!ctx().ok) return false; + uint32_t num_groups = (hdim + group_size - 1)/group_size, per = kv_heads*num_groups; + id scrI8 = recycled((size_t)(remaining?remaining:1)*kv_heads*hdim); + id scrSc = recycled((size_t)(remaining?remaining:1)*kv_heads*num_groups*sizeof(float)); + ensureEncoder(); + if (remaining > 0) { + uint32_t n = remaining*per; + [g_enc setComputePipelineState:ctx().psoSlideS]; + setBufAt(int8base, int8_bytes, 0); setBufAt(scalebase, scale_bytes, 1); + [g_enc setBuffer:scrI8 offset:0 atIndex:2]; [g_enc setBuffer:scrSc offset:0 atIndex:3]; + [g_enc setBytes:&kv_heads length:4 atIndex:4]; [g_enc setBytes:&hdim length:4 atIndex:5]; + [g_enc setBytes:&group_size length:4 atIndex:6]; [g_enc setBytes:&shift_src length:4 atIndex:7]; + [g_enc setBytes:&remaining length:4 atIndex:8]; + [g_enc dispatchThreads:MTLSizeMake(n,1,1) threadsPerThreadgroup:MTLSizeMake(n<256?n:256,1,1)]; + } + uint32_t n2 = (remaining+M)*per; + [g_enc setComputePipelineState:(M == 1 ? ctx().psoSlideR : ctx().psoSlideRM)]; + setBufAt(src, src_bytes, 0); setBufAt(int8base, int8_bytes, 1); setBufAt(scalebase, scale_bytes, 2); + [g_enc setBuffer:scrI8 offset:0 atIndex:3]; [g_enc setBuffer:scrSc offset:0 atIndex:4]; + [g_enc setBytes:&kv_heads length:4 atIndex:5]; [g_enc setBytes:&hdim length:4 atIndex:6]; + [g_enc setBytes:&group_size length:4 atIndex:7]; [g_enc setBytes:&keep_sink length:4 atIndex:8]; + [g_enc setBytes:&remaining length:4 atIndex:9]; + if (M != 1) [g_enc setBytes:&M length:4 atIndex:10]; + [g_enc dispatchThreads:MTLSizeMake(n2,1,1) threadsPerThreadgroup:MTLSizeMake(n2<256?n2:256,1,1)]; + return true; +} + +bool cactus_metal_encode_kv_append_sliding_i8(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, + uint32_t group_size, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + return encode_kv_append_sliding(src, int8base, scalebase, kv_heads, hdim, keep_sink, remaining, + shift_src, group_size, 1, src_bytes, int8_bytes, scale_bytes); +} + +bool cactus_metal_encode_kv_append_sliding_i8_m(const void* src, void* int8base, void* scalebase, + uint32_t kv_heads, uint32_t hdim, uint32_t keep_sink, uint32_t remaining, uint32_t shift_src, + uint32_t group_size, uint32_t M, size_t src_bytes, size_t int8_bytes, size_t scale_bytes) { + return encode_kv_append_sliding(src, int8base, scalebase, kv_heads, hdim, keep_sink, remaining, + shift_src, group_size, M, src_bytes, int8_bytes, scale_bytes); +} + +void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, + uint32_t M, __fp16* C) { + const uint32_t gs = W->group_size; + const bool fast = ctx().ok && M == 1 && quant_fast_eligible(W); + + if (!fast) { cactus_quant_matmul(W, A, M, C); return; } + + @autoreleasepool { + const uint32_t K = W->K, N = W->N, ng = W->num_groups; + const uint32_t pgb = (gs * 4u + 7u) / 8u; + + ResW& rw = resident(W); + if (!rw.ok) { cactus_quant_matmul(W, A, M, C); return; } + id bx = buf(A, (size_t)K * sizeof(__fp16)); + id brecip = rw.recip; + id bls = rw.lsign; + id brs = rw.rsign; + id bperm = rw.perm; + id bpk = rw.packed; + id bcb = rw.codebook; + id bnorm = rw.norms; + id bcode = owned_shared((size_t)ng * gs * sizeof(__fp16)); + id by = owned_shared((size_t)N * sizeof(__fp16)); + + id cmd = [ctx().queue commandBuffer]; + id e = [cmd computeCommandEncoder]; + + bool simdT = (gs==128u && ctx().psoTsimd); + [e setComputePipelineState:(simdT?ctx().psoTsimd:ctx().psoT)]; + [e setBuffer:bx offset:0 atIndex:0]; [e setBuffer:brecip offset:rw.rc_off atIndex:1]; + [e setBuffer:bls offset:rw.ls_off atIndex:2]; [e setBuffer:brs offset:rw.rs_off atIndex:3]; + [e setBuffer:bperm offset:rw.pm_off atIndex:4]; [e setBuffer:bcode offset:0 atIndex:5]; + [e setBytes:&gs length:sizeof(uint32_t) atIndex:6]; + [e setThreadgroupMemoryLength:gs * sizeof(float) atIndex:0]; + [e dispatchThreadgroups:MTLSizeMake(ng,1,1) threadsPerThreadgroup:MTLSizeMake(simdT?32u:(gs>1024u?1024u:gs),1,1)]; + + [e setComputePipelineState:ctx().psoG]; + [e setBuffer:bcode offset:0 atIndex:0]; [e setBuffer:bpk offset:rw.packed_off atIndex:1]; + [e setBuffer:bcb offset:rw.cb_off atIndex:2]; [e setBuffer:bnorm offset:rw.norms_off atIndex:3]; + [e setBuffer:by offset:0 atIndex:4]; + [e setBytes:&gs length:sizeof(uint32_t) atIndex:5]; + [e setBytes:&ng length:sizeof(uint32_t) atIndex:6]; + [e setBytes:&pgb length:sizeof(uint32_t) atIndex:7]; + [e setBytes:&N length:sizeof(uint32_t) atIndex:8]; + [e setBytes:&rw.il length:sizeof(uint32_t) atIndex:9]; + const uint32_t ROWS = 8; + [e dispatchThreadgroups:MTLSizeMake((N+ROWS-1)/ROWS,1,1) + threadsPerThreadgroup:MTLSizeMake(ROWS*32,1,1)]; + + [e endEncoding]; + [cmd commit]; + [cmd waitUntilCompleted]; + + std::memcpy(C, [by contents], (size_t)N * sizeof(__fp16)); + } +} + +#endif diff --git a/cactus-kernels/src/metal_backend_stub.cpp b/cactus-kernels/src/metal_backend_stub.cpp new file mode 100644 index 000000000..9ecafc05b --- /dev/null +++ b/cactus-kernels/src/metal_backend_stub.cpp @@ -0,0 +1,136 @@ + +#include "metal_backend.h" + +#if !CACTUS_HAS_METAL +bool cactus_metal_available() { return false; } +void cactus_metal_set_active(bool) {} +bool cactus_metal_active_mode() { return false; } +bool cactus_metal_encode_softcap(void*, const void*, size_t, float) { return false; } +bool cactus_metal_encode_adjust_logits(void*, size_t, const uint32_t*, uint32_t, + int64_t, float) { return false; } +bool cactus_metal_encode_transform_gemv(void*, const void*, const CactusQuantMatrix*, const void*) { return false; } +bool cactus_metal_transform_gemv_fits(uint32_t) { return false; } +bool cactus_metal_encode_gemv_cat(void* const*, const void* const*, + const CactusQuantMatrix* const*, int) { return false; } +bool cactus_metal_encode_swiglu_transform(void*, const void*, const void*, + const CactusQuantMatrix*, float) { return false; } +void cactus_metal_quant_matmul(const CactusQuantMatrix* W, const __fp16* A, + uint32_t M, __fp16* C) { + cactus_quant_matmul(W, A, M, C); +} +void cactus_metal_session_begin() {} +void cactus_metal_session_sync() {} +void cactus_metal_session_flush() {} +void cactus_metal_invalidate_host_wraps() {} +void cactus_metal_trim_prefill_cache() {} +void cactus_metal_session_end() {} +void* cactus_metal_alloc_shared(size_t) { return nullptr; } +void* cactus_metal_alloc_pooled(size_t) { return nullptr; } +void cactus_metal_free_shared(void*) {} +bool cactus_metal_encode_copy(void*, const void*, size_t) { return false; } +bool cactus_metal_encode_binary(int, void*, const void*, const void*, size_t) { return false; } +bool cactus_metal_encode_scalar(int, void*, const void*, size_t, float) { return false; } +bool cactus_metal_encode_unary(int, void*, const void*, size_t) { return false; } +bool cactus_metal_encode_swiglu(void*, const void*, const void*, size_t, float) { return false; } +bool cactus_metal_encode_rms_norm(void*, const void*, const void*, size_t, size_t, float) { return false; } +bool cactus_metal_encode_rms_norm_add(void*, const void*, const void*, const void*, size_t, size_t, float) { return false; } +bool cactus_metal_encode_rms_norm_add_rms(void*, void*, const void*, const void*, const void*, const void*, + size_t, size_t, float, float) { return false; } +bool cactus_metal_encode_rms_norm_add_scale(void*, const void*, const void*, const void*, size_t, size_t, float, float) { return false; } +bool cactus_metal_encode_argmax(const void*, uint32_t, void*, const void*) { return false; } +bool cactus_metal_encode_cast(void*, int, const void*, int, size_t) { return false; } +bool cactus_metal_encode_quant_matmul(void*, const void*, const CactusQuantMatrix*) { return false; } +bool cactus_metal_encode_quant_matmul_m(void*, const void*, const CactusQuantMatrix*, uint32_t) { return false; } +bool cactus_metal_encode_transform_batch(const void*, const CactusQuantMatrix* const*, int, void* const*) { return false; } +bool cactus_metal_encode_gemv_precoded(void*, const void*, const CactusQuantMatrix*) { return false; } +bool cactus_metal_prewarm_quant(const CactusQuantMatrix*) { return false; } +bool cactus_metal_encode_quant_matmul_ortho(void*, const void*, void*, const CactusQuantMatrix*) { return false; } +bool cactus_metal_encode_embedding_ortho(void*, uint32_t, const CactusQuantMatrix*, float) { return false; } +bool cactus_metal_encode_embedding_hadamard(void*, uint32_t, const CactusQuantMatrix*) { return false; } +bool cactus_metal_encode_embedding_ortho_m(void*, const CactusQuantMatrix*, const uint32_t*, uint32_t) { return false; } +bool cactus_metal_encode_embedding_hadamard_m(void*, const CactusQuantMatrix*, const uint32_t*, uint32_t) { return false; } +bool cactus_metal_encode_gather_f16(void*, const void*, size_t, const uint32_t*, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_attention_i8(void*, const void*, const void*, const void*, const void*, const void*, + const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, + float, size_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_attention_fused_i8(void*, const void*, const void*, const void*, + void*, void*, void*, void*, const void*, const void*, const void*, const void*, const void*, + uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, + float, float, size_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_attention_i8_prefill(void*, const void*, const void*, const void*, const void*, const void*, + const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, float, size_t, size_t, size_t, size_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_binary_f32(int, void*, const void*, const void*, size_t) { return false; } +bool cactus_metal_encode_scalar_f32(int, void*, const void*, size_t, float) { return false; } +bool cactus_metal_encode_unary_f32(int, void*, const void*, size_t) { return false; } +bool cactus_metal_encode_clamp(void*, const void*, size_t, float, float, int) { return false; } +bool cactus_metal_encode_glu(void*, const void*, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_layer_norm(void*, const void*, const void*, const void*, size_t, size_t, float) { return false; } +bool cactus_metal_encode_softmax_rows(void*, const void*, size_t, size_t) { return false; } +bool cactus_metal_encode_conv1d_k3(void*, const void*, const void*, int, const void*, uint32_t, + uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_gemm_f16(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_attention_f16(void*, const void*, const void*, const void*, const void*, + uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, + float, uint32_t, uint32_t, uint32_t, float, uint32_t) { return false; } +bool cactus_metal_encode_reduce_axis(int, void*, const void*, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_cumsum(void*, const void*, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_concat2(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_gather_f32idx(void*, const void*, const void*, uint32_t, uint32_t, size_t) { return false; } +bool cactus_metal_encode_rope_full(void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, int) { return false; } +bool cactus_metal_encode_maxpool1d(void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_bilinear(void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_conv1d_gen(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_conv1d_nlc_dw(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_conv2d(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_batchnorm(void*, const void*, const void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_groupnorm(void*, const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_bias_add_rows(void*, const void*, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_elemwise_chain(void*, const void*, const float*, uint32_t, const void*, const void*, const void*, const size_t*, size_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_rms_norm_add_rows(void*, void*, const void*, const void*, const void*, uint32_t, uint32_t, float, int) { return false; } +bool cactus_metal_encode_gemm_batch(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, int, int) { return false; } +bool cactus_metal_encode_conv1d_dw(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_transpose2d(void*, const void*, uint32_t, uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_strided_copy(void*, const void*, const uint32_t*, const uint32_t*, + uint32_t, uint32_t, uint32_t, size_t, size_t) { return false; } +bool cactus_metal_encode_bcast_binary(int, void*, const void*, const void*, const uint32_t*, + const uint32_t*, const uint32_t*, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_strided_scatter(void*, const void*, const uint32_t*, const uint32_t*, + uint32_t, uint32_t, uint32_t, size_t, size_t) { return false; } +bool cactus_metal_encode_kv_append_i8(const void*, void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_kv_append_sliding_i8(const void*, void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_kv_append_sliding_i8_m(const void*, void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_kv_append_i8_m(const void*, void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_kv_append_ring_i8_m(const void*, void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t, uint32_t, size_t, size_t, size_t) { return false; } +bool cactus_metal_encode_conv_cache_append(void*, const void*, void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_rel_pos_bias(void*, const void*, const void*, uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, int, float) { return false; } +bool cactus_metal_encode_gemv_bias(void*, const void*, const void*, const void*, + uint32_t, uint32_t, int) { return false; } +bool cactus_metal_encode_rope_pair(void*, const void*, const void*, const void*, + uint32_t, uint32_t) { return false; } +bool cactus_metal_encode_rope_pair_rms(void*, const void*, const void*, const void*, + const void*, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_deltanet_decode(void*, const void*, const void*, const void*, + const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_deltanet_prefill(void*, const void*, const void*, const void*, + const void*, const void*, const void*, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float) { return false; } +bool cactus_metal_encode_rms2_add_clip(void*, const void*, const void*, + const void*, const void*, size_t, float, float) { return false; } +bool cactus_metal_encode_rms_norm_scale(void*, const void*, const void*, + size_t, size_t, float, float) { return false; } +bool cactus_metal_encode_softmax_topk(void*, void*, const void*, + size_t, size_t, size_t, float) { return false; } +bool cactus_metal_encode_topk_rows(void*, const void*, size_t, size_t, size_t) { return false; } +bool cactus_metal_moe_cq4_ready(const CactusQuantMatrix*) { return false; } +bool cactus_metal_moe_cq4_build(const CactusQuantMatrix*, const CactusQuantMatrix*, + const CactusQuantMatrix*, uint32_t) { return false; } +bool cactus_metal_encode_moe_gated_cq4(void*, const void*, const void*, const void*, const CactusQuantMatrix*, + uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, float) { return false; } +#endif diff --git a/docs/cactus_engine.md b/docs/cactus_engine.md index 6d2f20d7b..00d8dc6e4 100644 --- a/docs/cactus_engine.md +++ b/docs/cactus_engine.md @@ -25,9 +25,10 @@ cactus auth `cactus download` fetches a **pre-built runtime bundle** (CQ weights + serialized graph + manifest) from [huggingface.co/Cactus-Compute](https://huggingface.co/Cactus-Compute) into -`weights/-cq[-]/`. Defaults to `--platform auto` (the -best build for your host, e.g. Apple Silicon on macOS); pass `--platform cpu` -for the generic build. The result can be loaded directly via `cactus_init()`. +`weights/-cq[-]/`. Defaults to `--weights general` (the +portable build, used on every platform); pass `--weights apple` for the +Apple-specific (CoreML/NPU) variant. The result can be loaded directly via +`cactus_init()`. For models not on Cactus-Compute, build a bundle from source with `cactus convert ` (quantizes the weights and builds the runtime graph). diff --git a/docs/cactus_transpiler.md b/docs/cactus_transpiler.md index 02c128a72..3be3c77ef 100644 --- a/docs/cactus_transpiler.md +++ b/docs/cactus_transpiler.md @@ -454,7 +454,7 @@ huggingface.co/Cactus-Compute) or a local path to a bundle directory. | Option | Description | |--------|-------------| | `--bits 1\|2\|3\|4` | CQ quantization bits when downloading (default: 4) | -| `--platform auto\|cpu\|apple` | Target platform (default: auto — best for the host) | +| `--weights general\|apple` | Weights bundle variant (default: general — portable on every platform) | | `--token ` | HuggingFace token (gated models) | | `--prompt ` | Input prompt | | `--input-ids ` | Comma-separated token IDs | diff --git a/python/README.md b/python/README.md index a259aa8ce..6bf7e0dff 100644 --- a/python/README.md +++ b/python/README.md @@ -20,9 +20,9 @@ cactus build --python ```bash -# Download pre-built bundles (defaults to --platform auto; use --platform cpu for the generic build) +# Download pre-built bundles (defaults to --weights general, the portable build) cactus download LiquidAI/LFM2-VL-450M -cactus download openai/whisper-small --platform apple # CoreML/NPU variant +cactus download openai/whisper-small --weights apple # CoreML/NPU variant # Optional: set your Cactus Cloud API key for automatic cloud fallback cactus auth diff --git a/python/cactus/__init__.py b/python/cactus/__init__.py index 3f791dd07..585fe7bf1 100644 --- a/python/cactus/__init__.py +++ b/python/cactus/__init__.py @@ -15,6 +15,7 @@ "Graph", "Tensor", "cactus_init", + "cactus_set_backend", "cactus_destroy", "cactus_reset", "cactus_stop", diff --git a/python/cactus/bindings/cactus.py b/python/cactus/bindings/cactus.py index 27e85f84d..5eae1feee 100644 --- a/python/cactus/bindings/cactus.py +++ b/python/cactus/bindings/cactus.py @@ -67,6 +67,9 @@ class cactus_tensor_info_t(ctypes.Structure): _lib.cactus_init.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_bool] _lib.cactus_init.restype = ctypes.c_void_p +_lib.cactus_set_backend.argtypes = [ctypes.c_char_p] +_lib.cactus_set_backend.restype = ctypes.c_int + # cactus graph API _lib.cactus_graph_create.restype = cactus_graph_t _lib.cactus_graph_destroy.argtypes = [cactus_graph_t] @@ -952,6 +955,11 @@ def cactus_telemetry_shutdown(): # ── Model lifecycle ────────────────────────────────────────────────── +def cactus_set_backend(backend): + """Select the inference backend ("auto", "cpu", or "metal"). Returns 0 on success, -1 on failure.""" + return _lib.cactus_set_backend(_enc(backend)) + + def cactus_init(model_path, corpus_dir=None, cache_index=False): """Load a model from disk. @@ -1481,6 +1489,7 @@ class Graph: CQ4 = 6 CPU = 0 NPU = 1 + METAL = 2 ACT_SILU = 0 ACT_GELU = 1 ACT_GELU_ERF = 2 diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 0e01978b7..708477609 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -7,17 +7,16 @@ DEFAULT_TRANSCRIPTION_MODEL_ID, DEFAULT_TEST_MODEL_ID, DEFAULT_TEST_TRANSCRIPTION_MODEL_ID, - SUPPORTED_PLATFORMS, + SUPPORTED_WEIGHTS_VARIANTS, ) from .download import cmd_download -_PLATFORM_CHOICES = ("auto", "cpu", *SUPPORTED_PLATFORMS) -_PLATFORM_HELP = ( - f"target platform: auto = best for this host, e.g. apple on macOS (default); " - f"cpu = generic ARM; " - f"or one of: {', '.join(SUPPORTED_PLATFORMS) if SUPPORTED_PLATFORMS else '(none yet)'}" +_WEIGHTS_CHOICES = ("general", *SUPPORTED_WEIGHTS_VARIANTS) +_WEIGHTS_HELP = ( + f"weights bundle variant: general = portable bundle for every platform (default); " + f"or a vendor variant: {', '.join(SUPPORTED_WEIGHTS_VARIANTS) if SUPPORTED_WEIGHTS_VARIANTS else '(none yet)'}" ) -_PLATFORM_PIPE = "|".join(_PLATFORM_CHOICES) +_WEIGHTS_PIPE = "|".join(_WEIGHTS_CHOICES) from .compile import cmd_build from .serve import cmd_serve from .transcribe import cmd_transcribe @@ -45,8 +44,8 @@ def _build_parent(): p = argparse.ArgumentParser(add_help=False) p.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, help="CQ quantization (default: 4)") - p.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help=_PLATFORM_HELP) + p.add_argument("--weights", choices=_WEIGHTS_CHOICES, default="general", + help=_WEIGHTS_HELP) p.add_argument("--token", help="HuggingFace token") p.add_argument("--reconvert", action="store_true", help="Force local rebuild from source") @@ -111,7 +110,7 @@ def create_parser(): cactus run [model|path] run a model (default: {DEFAULT_MODEL_ID}) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --image image file for VLM inference --audio audio file for audio chat --system system prompt @@ -124,19 +123,19 @@ def create_parser(): --file audio file to transcribe (WAV) --language language code (default: en) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (gated models) --reconvert force local rebuild from source cactus download [model] fetch a prebuilt bundle, else build locally (default: {DEFAULT_MODEL_ID}) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (gated models) --reconvert force local rebuild from source cactus convert [dir] build a runnable bundle locally (skips prebuilt fetch) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (gated models) --reconvert force local rebuild from source --lora merge a LoRA adapter before converting @@ -145,7 +144,7 @@ def create_parser(): --host bind address (default: 127.0.0.1) --port port (default: 8080) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (gated models) --reconvert force local rebuild from source --no-cloud-handoff disable automatic cloud handoff @@ -165,7 +164,7 @@ def create_parser(): --model default: {DEFAULT_TEST_MODEL_ID} --transcription-model default: {DEFAULT_TEST_TRANSCRIPTION_MODEL_ID} --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (gated models) --reconvert force local rebuild of test models --suite run a single test suite from any @@ -189,7 +188,7 @@ def create_parser(): (quantizes weights to CQ, then builds the runtime graph) --bits 1|2|3|4 CQ quantization (default: 4) - --platform {_PLATFORM_PIPE:<22} target platform (default: auto) + --weights {_WEIGHTS_PIPE:<23} weights bundle variant (default: general) --token HuggingFace token (defaults to $HF_TOKEN) --reconvert force weight conversion from source --lora merge a LoRA adapter before converting @@ -285,6 +284,8 @@ def create_parser(): help="Confidence threshold below which local completions may hand off to cloud") run_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, help="Maximum time to wait for cloud handoff before falling back locally") + run_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", + help="Inference backend, default auto (Metal on Apple Silicon GPUs)") transcribe_parser = subparsers.add_parser("transcribe", help="Transcribe audio with a model", parents=[_telemetry_parent(), _build_parent()]) @@ -313,6 +314,8 @@ def create_parser(): help="Maximum time to wait for cloud handoff before falling back locally") serve_parser.add_argument("--no-access-log", action="store_true", help="Disable per-request HTTP access logging") + serve_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", + help="Inference backend, default auto (Metal on Apple Silicon GPUs)") code_parser = subparsers.add_parser("code", help="Run the Cactus coding agent (TUI / print mode)", parents=[_build_parent()]) @@ -330,6 +333,8 @@ def create_parser(): help="Confidence threshold below which completions hand off to cloud") code_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, help="Maximum time to wait for cloud handoff before falling back locally") + code_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", + help="Inference backend, default auto (Metal on Apple Silicon GPUs)") code_parser.add_argument("agent_args", nargs=argparse.REMAINDER, help="Arguments passed through to the coding agent (prefix with -- )") @@ -337,6 +342,8 @@ def create_parser(): parents=[_build_parent()]) test_parser.add_argument("--component", choices=COMPONENTS, default="all", help="Component to test (default: all)") + test_parser.add_argument("--backend", choices=["auto", "cpu", "metal"], default="auto", + help="Inference backend, default auto (Metal on Apple Silicon GPUs)") test_parser.add_argument("--model", dest="model_id", default=None, type=_hf_id_or_path, help=f"HF model ID under test (default: {DEFAULT_TEST_MODEL_ID})") @@ -371,8 +378,8 @@ def create_parser(): help="Output directory (default: weights/)") convert_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, help="CQ quantization bits (default: 4)") - convert_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="auto", - help=_PLATFORM_HELP) + convert_parser.add_argument("--weights", choices=_WEIGHTS_CHOICES, default="general", + help=_WEIGHTS_HELP) convert_parser.add_argument("--token", help="HuggingFace token") convert_parser.add_argument("--lora", help="Path or HF id of a LoRA adapter to merge before converting (requires `peft`)") diff --git a/python/cactus/cli/code.py b/python/cactus/cli/code.py index 87d023a20..b8362ebf2 100644 --- a/python/cactus/cli/code.py +++ b/python/cactus/cli/code.py @@ -130,7 +130,7 @@ def cmd_code(args) -> int: model_flags_given = ( getattr(args, "reconvert", False) or getattr(args, "bits", 4) != 4 - or getattr(args, "platform", "auto") != "auto" + or getattr(args, "weights", "general") != "general" or bool(getattr(args, "token", None)) ) if args.serve_model: @@ -146,7 +146,8 @@ def cmd_code(args) -> int: sys.executable, "-m", "cactus", "serve", serve_model, "--host", args.host, "--port", str(args.port), "--no-access-log", "--bits", str(getattr(args, "bits", 4)), - "--platform", getattr(args, "platform", "auto"), + "--weights", getattr(args, "weights", "general"), + "--backend", getattr(args, "backend", "auto"), ] if getattr(args, "token", None): serve_cmd += ["--token", args.token] diff --git a/python/cactus/cli/common.py b/python/cactus/cli/common.py index 71601e4cb..345dc8e81 100644 --- a/python/cactus/cli/common.py +++ b/python/cactus/cli/common.py @@ -35,12 +35,13 @@ def transpiled_root() -> Path: DEFAULT_MODEL_ID = "google/gemma-4-E2B-it" DEFAULT_TRANSCRIPTION_MODEL_ID = "nvidia/parakeet-tdt-0.6b-v3" -DEFAULT_TEST_MODEL_ID = "LiquidAI/LFM2-VL-450M" -DEFAULT_TEST_TRANSCRIPTION_MODEL_ID = "openai/whisper-base" +DEFAULT_TEST_MODEL_ID = DEFAULT_MODEL_ID +DEFAULT_TEST_TRANSCRIPTION_MODEL_ID = DEFAULT_TRANSCRIPTION_MODEL_ID -# Add a new vendor platform by appending its name here. -SUPPORTED_PLATFORMS: tuple[str, ...] = ("apple",) +# Vendor-specific weight-bundle variants; add a new vendor by appending its +# name here. "general" (the portable bundle) is always available and default. +SUPPORTED_WEIGHTS_VARIANTS: tuple[str, ...] = ("apple",) RED = '\033[0;31m' diff --git a/python/cactus/cli/compile.py b/python/cactus/cli/compile.py index c5193cd06..429cbdb39 100644 --- a/python/cactus/cli/compile.py +++ b/python/cactus/cli/compile.py @@ -99,6 +99,8 @@ def build_binary( "-framework", "Accelerate", "-framework", "CoreML", "-framework", "Foundation", + "-framework", "Metal", + "-framework", "MetalPerformanceShaders", "-framework", "Security", "-framework", "SystemConfiguration", "-framework", "CFNetwork", diff --git a/python/cactus/cli/convert.py b/python/cactus/cli/convert.py index 55cf3333e..6b9191d26 100644 --- a/python/cactus/cli/convert.py +++ b/python/cactus/cli/convert.py @@ -3,7 +3,7 @@ from pathlib import Path from .common import GREEN, RED, YELLOW, print_color -from .download import get_bundle_dir, get_weights_dir, resolve_platform +from .download import get_bundle_dir, get_weights_dir, resolve_weights_variant def _merge_lora_adapter(base_model_id, lora_path, token=None): @@ -56,7 +56,7 @@ def cmd_convert(args): return 1 source_model_id = merged_dir - platform = resolve_platform(getattr(args, "platform", "auto")) + platform = resolve_weights_variant(getattr(args, "weights", "general")) output_dir = args.output_dir or str( get_bundle_dir(args.model_id, bits=args.bits, platform=platform) ) diff --git a/python/cactus/cli/download.py b/python/cactus/cli/download.py index 67beb9261..14192e34b 100644 --- a/python/cactus/cli/download.py +++ b/python/cactus/cli/download.py @@ -3,7 +3,7 @@ from .common import ( BLUE, GREEN, - SUPPORTED_PLATFORMS, + SUPPORTED_WEIGHTS_VARIANTS, print_color, weights_root, ) @@ -21,23 +21,15 @@ def get_bundle_dir(model_id: str, *, bits: int = 4, platform: str | None = None) return weights_root() / f"{get_model_dir_name(model_id)}-{variant_suffix(bits, platform)}" -def _host_platform() -> str | None: - """Best-fit platform for the current host: apple on macOS, else generic CPU.""" - import platform as _platform - - if _platform.system() == "Darwin" and "apple" in SUPPORTED_PLATFORMS: - return "apple" - return None - - -def resolve_platform(choice: str) -> str | None: - if choice == "auto": - return _host_platform() - if choice == "cpu": +def resolve_weights_variant(choice: str | None) -> str | None: + """Map the --weights choice to the bundle variant suffix: general (the + default on every platform) selects the portable bundle (no suffix).""" + if choice in (None, "general"): return None - if choice in SUPPORTED_PLATFORMS: + if choice in SUPPORTED_WEIGHTS_VARIANTS: return choice - raise ValueError(f"unknown platform {choice!r}; supported: auto, cpu, {', '.join(SUPPORTED_PLATFORMS)}") + raise ValueError(f"unknown weights variant {choice!r}; " + f"supported: general, {', '.join(SUPPORTED_WEIGHTS_VARIANTS)}") def ensure_model(model_id: str) -> Path: diff --git a/python/cactus/cli/model.py b/python/cactus/cli/model.py index 0bcf8ab81..483067597 100644 --- a/python/cactus/cli/model.py +++ b/python/cactus/cli/model.py @@ -238,15 +238,15 @@ def ensure_runnable_bundle(model_id, *, bits=4, platform=None, token=None, def prepare_bundle(args, *, model_id=None, transpile=None, prebuilt=True, output_dir=None, fail_prefix="Model setup failed"): - """Resolve the platform from args and return a runnable bundle, with uniform + """Resolve the weights variant from args and return a runnable bundle, with uniform error handling shared by every model command. Returns the bundle Path, or None (after printing the error) on failure.""" - from .download import resolve_platform + from .download import resolve_weights_variant try: return ensure_runnable_bundle( args.model_id if model_id is None else model_id, bits=getattr(args, "bits", 4), - platform=resolve_platform(getattr(args, "platform", "auto")), + platform=resolve_weights_variant(getattr(args, "weights", "general")), token=getattr(args, "token", None), reconvert=getattr(args, "reconvert", False), prebuilt=prebuilt, diff --git a/python/cactus/cli/run.py b/python/cactus/cli/run.py index e7bd3814a..38078a3a8 100644 --- a/python/cactus/cli/run.py +++ b/python/cactus/cli/run.py @@ -40,6 +40,7 @@ def cmd_run(args) -> int: ("--result-json", args.result_json), ("--confidence-threshold", args.confidence_threshold), ("--cloud-timeout-ms", args.cloud_timeout_ms), + ("--backend", args.backend), ): if value is not None: cmd.extend([flag, str(value)]) diff --git a/python/cactus/cli/runtime.py b/python/cactus/cli/runtime.py index 9e162cddb..54956a7c4 100644 --- a/python/cactus/cli/runtime.py +++ b/python/cactus/cli/runtime.py @@ -77,6 +77,8 @@ def _link_python_runtime_library(*, static_lib, library_path): "-framework", "Accelerate", "-framework", "CoreML", "-framework", "Foundation", + "-framework", "Metal", + "-framework", "MetalPerformanceShaders", "-framework", "Security", "-framework", "SystemConfiguration", "-framework", "CFNetwork", diff --git a/python/cactus/cli/serve.py b/python/cactus/cli/serve.py index 5f587ef9e..67ee60e11 100644 --- a/python/cactus/cli/serve.py +++ b/python/cactus/cli/serve.py @@ -54,6 +54,12 @@ def cmd_serve(args): print_color(RED, "Error: uvicorn not installed. Install the serve extra or run `pip install fastapi uvicorn python-multipart`.") return 1 + backend = getattr(args, "backend", "auto") + if backend and backend != "auto": + from ..bindings.cactus import cactus_set_backend + if cactus_set_backend(backend) != 0: + print_color(YELLOW, f"Backend '{backend}' not available; using auto") + try: from ..server import create_app except ImportError: diff --git a/python/cactus/cli/test.py b/python/cactus/cli/test.py index 8c64ba7ea..82b43fa6f 100644 --- a/python/cactus/cli/test.py +++ b/python/cactus/cli/test.py @@ -31,6 +31,7 @@ def _component_args(component, args): if component == "engine": cmd.extend(["--model", args.model_id]) cmd.extend(["--transcription-model", args.transcription_model_id]) + cmd.extend(["--backend", getattr(args, "backend", "auto")]) if args.android: cmd.append("--android") if args.ios: diff --git a/python/cactus/cli/upload.py b/python/cactus/cli/upload.py index b0277b737..2a9e9486f 100644 --- a/python/cactus/cli/upload.py +++ b/python/cactus/cli/upload.py @@ -4,7 +4,7 @@ from pathlib import Path from .common import BLUE, GREEN, PROJECT_ROOT, print_color -from .download import get_model_dir_name, resolve_platform +from .download import get_model_dir_name, resolve_weights_variant from .model import prepare_bundle from .utils import suggested_cq_repo, variant_suffix @@ -15,7 +15,7 @@ def cmd_upload(args): if not token: raise SystemExit("missing HuggingFace token: pass --token or set HF_TOKEN") - platform = resolve_platform(args.platform) + platform = resolve_weights_variant(getattr(args, "weights", "general")) repo_id = suggested_cq_repo(args.model_id) stem = f"{get_model_dir_name(args.model_id)}-{variant_suffix(args.bits, platform)}" archive_name = f"{stem}.zip" diff --git a/python/cactus/transpile/model_adapters.py b/python/cactus/transpile/model_adapters.py index 32f322b2a..fb6f09988 100644 --- a/python/cactus/transpile/model_adapters.py +++ b/python/cactus/transpile/model_adapters.py @@ -223,16 +223,25 @@ def _tile_to_length(tensor: torch.Tensor, length: int) -> torch.Tensor: def _resolve_model_pad_token_id(model: torch.nn.Module) -> int | None: + def _coerce(value: object) -> int | None: + if isinstance(value, (list, tuple)): + value = value[0] if value else None + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None config = getattr(model, "config", None) for attr_name in ("pad_token_id", "eos_token_id", "bos_token_id"): - value = getattr(config, attr_name, None) - if value is not None: - return int(value) + resolved = _coerce(getattr(config, attr_name, None)) + if resolved is not None: + return resolved generation_config = getattr(model, "generation_config", None) for attr_name in ("pad_token_id", "eos_token_id", "bos_token_id"): - value = getattr(generation_config, attr_name, None) - if value is not None: - return int(value) + resolved = _coerce(getattr(generation_config, attr_name, None)) + if resolved is not None: + return resolved return None @@ -1162,6 +1171,7 @@ def _gemma4_text_attention_forward( hidden_states=hidden_states, position_embeddings=position_embeddings, attention_mask=attention_mask, + shared_kv_states={}, position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, @@ -4281,6 +4291,138 @@ def _build_gemma3_causal_lm_component_specs( return specs +def _build_gemma4_causal_lm_component_specs( + model: torch.nn.Module, + *, + named_tensors: dict[str, torch.Tensor], + weights_dir: str | None = None, + components: tuple[str, ...] | None = None, + cache_context_length: str | int | None = None, + dynamic_batch: bool = True, + max_slots: int = 1, +) -> list[ComponentModuleSpec] | None: + input_ids = named_tensors.get("input_ids") + if input_ids is None: + return None + pad_token_id = _resolve_model_pad_token_id(model) + requested_set = set(components or ()) + if not requested_set or "decoder" in requested_set: + requested_set |= { + "decoder", + "lm_encoder_step", + "lm_encoder_text_chunk", + "decoder_prefill_chunk", + "decoder_step", + } + common_graph_meta = { + "weights_dir": weights_dir, + "task": "causal_lm_logits", + "adapter_family": "gemma4", + } + metadata = {"family": "gemma4", "task": "causal_lm_logits"} + + specs: list[ComponentModuleSpec] = [] + if "decoder" in requested_set: + decoder = Gemma4CausalLMLogitsAdapter(model, pad_token_id=pad_token_id).eval() + specs.append(ComponentModuleSpec( + component="decoder", + module=decoder, + example_inputs=(input_ids,), + input_keys=("input_ids",), + output_keys=("logits",), + graph_meta={**common_graph_meta, "component": "decoder"}, + metadata=metadata, + )) + + cached_components = {"lm_encoder_step", "lm_encoder_text_chunk", "decoder_prefill_chunk", "decoder_step"} + if not (cached_components & requested_set): + return specs + + lm_encoder_step = Gemma4LMEncoderStepAdapter(model, weights_dir=weights_dir).eval() + lm_encoder_text_chunk = Gemma4LMEncoderTextChunkAdapter(model, weights_dir=weights_dir).eval() + decoder_prefill_chunk = Gemma4DecoderPrefillChunkAdapter(model, weights_dir=weights_dir).eval() + decoder_step = Gemma4DecoderStepAdapter(model, weights_dir=weights_dir).eval() + cache_seq_len = _max_cache_seq_len(model, input_ids, cache_context_length, fallback_extra_tokens=512) + prefill_chunk_size = max(2, int(os.environ.get("CACTUS_GEMMA4_PREFILL_CHUNK", "128") or "128")) + + step_input_ids = input_ids[:, -1:].contiguous() + step_position_ids = torch.full( + (int(input_ids.shape[0]), 1), + max(0, int(input_ids.shape[1]) - 1), + dtype=torch.long, + device=input_ids.device, + ) + chunk_input_ids = _tile_to_length(input_ids, prefill_chunk_size) + chunk_position_ids = torch.arange( + prefill_chunk_size, dtype=torch.long, device=input_ids.device, + ).unsqueeze(0).expand(int(input_ids.shape[0]), -1).contiguous() + with torch.no_grad(): + decoder_step_inputs = lm_encoder_step(step_input_ids, step_position_ids) + prefill_decoder_inputs = lm_encoder_text_chunk(chunk_input_ids, chunk_position_ids) + + if "lm_encoder_step" in requested_set: + specs.append(ComponentModuleSpec( + component="lm_encoder_step", + module=lm_encoder_step, + example_inputs=(step_input_ids, step_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + graph_meta={**common_graph_meta, "component": "lm_encoder_step"}, + metadata=metadata, + dynamic_batch_axis=0 if dynamic_batch else None, + )) + if "lm_encoder_text_chunk" in requested_set: + specs.append(ComponentModuleSpec( + component="lm_encoder_text_chunk", + module=lm_encoder_text_chunk, + example_inputs=(chunk_input_ids, chunk_position_ids), + input_keys=("input_ids", "position_ids"), + output_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + graph_meta={ + **common_graph_meta, + "component": "lm_encoder_text_chunk", + "encoder_chunk_size": prefill_chunk_size, + }, + metadata=metadata, + )) + if "decoder_prefill_chunk" in requested_set: + specs.append(ComponentModuleSpec( + component="decoder_prefill_chunk", + module=decoder_prefill_chunk, + example_inputs=tuple(prefill_decoder_inputs), + input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + output_keys=("logits",), + graph_meta={ + **common_graph_meta, + "component": "decoder_prefill_chunk", + "use_internal_kv_cache": True, + "max_cache_seq_len": cache_seq_len, + "cache_sink_size": 4, + "prefill_chunk_size": prefill_chunk_size, + }, + metadata=metadata, + )) + if "decoder_step" in requested_set: + specs.append(ComponentModuleSpec( + component="decoder_step", + module=decoder_step, + example_inputs=tuple(decoder_step_inputs), + input_keys=_GEMMA4_DECODER_PIPELINE_IO_KEYS, + output_keys=("logits", "probe_hidden"), + graph_meta={ + **common_graph_meta, + "component": "decoder_step", + "use_internal_kv_cache": True, + "max_cache_seq_len": cache_seq_len, + "cache_sink_size": 4, + "cache_num_slots": (max_slots if dynamic_batch else 1), + }, + metadata=metadata, + dynamic_batch_axis=0 if dynamic_batch else None, + )) + return specs + + def _gemma4_make_audio_mask_fp16_safe(model: torch.nn.Module) -> None: backbone = getattr(model, "model", model) audio_tower = getattr(backbone, "audio_tower", None) @@ -6978,6 +7120,16 @@ def build_component_module_specs( dynamic_batch=dynamic_batch, max_slots=max_slots, ) + if family == "gemma4" and task == "causal_lm_logits": + return _build_gemma4_causal_lm_component_specs( + model, + named_tensors=named_tensors, + weights_dir=weights_dir, + components=components, + cache_context_length=cache_context_length, + dynamic_batch=dynamic_batch, + max_slots=max_slots, + ) if family == "gemma4" and task == "multimodal_causal_lm_logits": return _build_gemma4_multimodal_component_specs( model, diff --git a/python/tests/bundles.py b/python/tests/bundles.py index 058620410..00f1548c9 100644 --- a/python/tests/bundles.py +++ b/python/tests/bundles.py @@ -27,7 +27,7 @@ def _iter_bundle_candidates(name: str): if bare.exists(): yield bare if WEIGHTS.is_dir(): - for candidate in sorted(WEIGHTS.glob(f"{name}-cq*")): + for candidate in sorted(WEIGHTS.glob(f"{name}-cq*"), reverse=True): if candidate.is_dir(): yield candidate diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py index 818e8275b..79ad255dd 100644 --- a/python/tests/test_bindings.py +++ b/python/tests/test_bindings.py @@ -336,14 +336,14 @@ def test_run_rejects_bare_name(self): with pytest.raises(SystemExit): self.parser.parse_args(["run", "whisper-base", "--prompt", "hi"]) - def test_run_platform_flag(self): - args = self.parser.parse_args(["run", "Foo/Bar", "--platform", "apple"]) - assert args.platform == "apple" + def test_run_weights_flag(self): + args = self.parser.parse_args(["run", "Foo/Bar", "--weights", "apple"]) + assert args.weights == "apple" assert args.bits == 4 - def test_download_platform_flag(self): - args = self.parser.parse_args(["download", "Foo/Bar", "--platform", "cpu", "--bits", "2"]) - assert args.platform == "cpu" + def test_download_weights_flag(self): + args = self.parser.parse_args(["download", "Foo/Bar", "--weights", "general", "--bits", "2"]) + assert args.weights == "general" assert args.bits == 2 diff --git a/python/tests/test_cli_run.py b/python/tests/test_cli_run.py index 5c4c0d7e5..97aa6a59c 100644 --- a/python/tests/test_cli_run.py +++ b/python/tests/test_cli_run.py @@ -39,7 +39,7 @@ def fake_subprocess_run(cmd): no_cloud_tele=False, model_id="org/model", bits=4, - platform="cpu", + weights="general", token=None, reconvert=False, system=None, @@ -52,6 +52,7 @@ def fake_subprocess_run(cmd): result_json=str(result_json), confidence_threshold=None, cloud_timeout_ms=None, + backend="auto", thinking=False, no_cloud_handoff=False, ) diff --git a/python/tests/test_model.py b/python/tests/test_model.py index b6b859ad7..3addbffd7 100644 --- a/python/tests/test_model.py +++ b/python/tests/test_model.py @@ -17,10 +17,10 @@ cactus_transcribe, ) from cactus.cli.model import ensure_bundle -from cactus.cli.download import resolve_platform +from cactus.cli.download import resolve_weights_variant -_PLATFORM = resolve_platform("auto") +_PLATFORM = resolve_weights_variant("general") _ASSETS_DIR = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" _TEST_IMAGE = _ASSETS_DIR / "test_monkey.png" _TEST_AUDIO = _ASSETS_DIR / "test.wav"